diff --git a/.env.example b/.env.example index 23e0fdaa5..51fc45f67 100644 --- a/.env.example +++ b/.env.example @@ -33,18 +33,6 @@ DATABASE_SSL=false # DATABASE_PLATFORM_USERNAME=voidhash # DATABASE_PLATFORM_PASSWORD=replace-with-a-random-password # DATABASE_PLATFORM_SSL=false -# Optional analytics profile. Leave CLICKHOUSE_URL unset for the core stack. -# CLICKHOUSE_URL=http://clickhouse:8123 -CLICKHOUSE_DATABASE=voidhash -CLICKHOUSE_ADMIN_USERNAME=voidhash_admin -CLICKHOUSE_ADMIN_PASSWORD=replace-with-a-random-password -CLICKHOUSE_USERNAME=voidhash_app -CLICKHOUSE_PASSWORD=replace-with-a-random-password -CLICKHOUSE_RO_USERNAME=voidhash_ro -CLICKHOUSE_RO_PASSWORD=replace-with-a-random-password -CLICKHOUSE_ANALYTICS_QUERY_USERNAME=voidhash_query -CLICKHOUSE_ANALYTICS_QUERY_PASSWORD=replace-with-a-random-password -CLICKHOUSE_HTTP_PORT=8123 MIMIC_ROOT_USERNAME=root MIMIC_ROOT_PASSWORD=replace-with-a-random-password PUBLIC_BASE_URL=http://localhost:5001 @@ -101,7 +89,7 @@ MAILPIT_UI_PORT=8025 # ── Local development & integration tests ──────────────────────────────────── # Used together with docker-compose.dev.yml: # docker compose -f docker-compose.yml -f docker-compose.dev.yml \ -# --profile analytics up -d --build +# up -d --build # `pnpm test:integration` (repo root) reads this file and derives host-side # connection settings from the values below, so the whole suite runs against # this stack with no additional configuration. @@ -111,8 +99,6 @@ MAILPIT_UI_PORT=8025 DATABASE_HOST_PORT=5432 COMPILER_HOST_PORT=5002 -# To enable the analytics profile end-to-end (the compose service, migrations, -# and the ClickHouse integration suite), uncomment CLICKHOUSE_URL above. # Browser used by the screenshot integration tests on the host. The container # ships its own chromium; this is only for host-side test runs. diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index 0275e740b..7bd1110ec 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -63,20 +63,15 @@ jobs: grep -E '^[A-Z][A-Z0-9_]*=' .env >> "$GITHUB_ENV" - name: Start stateful stores - run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env --profile analytics up -d clickhouse minio --wait --wait-timeout 180 + run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env up -d minio --wait --wait-timeout 180 - name: Initialize object store - run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env --profile analytics run --rm minio-init + run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env run --rm minio-init - # The dev overlay publishes Postgres and the compiler, which the - # host-side integration tier connects to. `CLICKHOUSE_URL` stays a shell - # override rather than an entry in `.env`: it names the compose-internal - # endpoint the application dials, while every host-side tier reaches - # ClickHouse on the published port instead. + # The dev overlay publishes PostgreSQL and the compiler, which the + # host-side integration tier connects to. - name: Build and start Community Compose - env: - CLICKHOUSE_URL: http://clickhouse:8123 - run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env --profile analytics up --build --wait --wait-timeout 180 + run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env up --build --wait --wait-timeout 180 - name: Reclaim image build cache run: docker builder prune --all --force @@ -100,9 +95,9 @@ jobs: - name: Show Compose diagnostics if: always() run: | - docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --profile analytics ps || true - docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --profile analytics logs --no-color || true + docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml ps || true + docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml logs --no-color || true - name: Stop Compose if: always() - run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --profile analytics down --volumes --remove-orphans + run: docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml down --volumes --remove-orphans diff --git a/apps/backend/package.json b/apps/backend/package.json index ca46325b8..c3faec8b8 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -28,7 +28,6 @@ "@voidhash/api-contracts": "workspace:*", "@voidhash/app-store-server-sdk": "workspace:*", "@voidhash/backend": "workspace:*", - "@voidhash/clickhouse-db": "workspace:*", "@voidhash/core": "workspace:*", "@voidhash/db": "workspace:*", "@voidhash/lib": "workspace:*", diff --git a/apps/backend/src/backend/Analytics.ts b/apps/backend/src/backend/Analytics.ts index 10cfcb1d1..6a90fb13c 100644 --- a/apps/backend/src/backend/Analytics.ts +++ b/apps/backend/src/backend/Analytics.ts @@ -1,300 +1,21 @@ -import { - AnalyticsIngestQueueMessage, - type AnalyticsIngestQueueMessageType, - type AnalyticsWriterMessageType, -} from "@voidhash/core/domain/analyticsIngest/AnalyticsIngest"; -import { IdentityProjectionPublisher } from "@voidhash/core/services"; -import { AnalyticsIngestDlqService } from "@voidhash/core/services/analyticsIngest/AnalyticsIngestDlqService"; +import { AnalyticsEventStore } from "@voidhash/core/services/analytics/AnalyticsEventStore"; import { AnalyticsDispatchService } from "@voidhash/core/services/analyticsIngest/AnalyticsDispatchService"; -import { AnalyticsWriterService } from "@voidhash/core/services/analyticsIngest/AnalyticsWriterService"; -import { - CaptureIngress, - CaptureIngressError, - type PublishableCaptureEvent, -} from "@voidhash/core/services/analyticsIngest/CaptureIngress"; -import { DlqProducer } from "@voidhash/core/services/analyticsIngest/DlqProducer"; import { EventCaptureService } from "@voidhash/core/services/analyticsIngest/EventCaptureService"; -import { EventProcessorService } from "@voidhash/core/services/analyticsIngest/EventProcessorService"; -import { - PolicyCounterStore, - PolicyStoreError, -} from "@voidhash/core/services/analyticsIngest/PolicyCounterStore"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { PersonIdentityService } from "@voidhash/core/services/personIdentity/PersonIdentityService"; import { Db } from "@voidhash/db"; -import { KeyValueStore } from "@voidhash/platform/KeyValueStore"; -import { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; -import { QueueDriver } from "@voidhash/platform/Queue"; -import { Context, Effect, Layer, Schema } from "effect"; +import { Layer } from "effect"; import type { SelfhostRuntimeConfig } from "../config.ts"; -import { makeSelfhostPlatformLive } from "./PlatformProfile.ts"; -const analyticsQueueName = "analytics-ingest"; -const analyticsDeadLetterQueueName = "analytics-ingest-dlq"; - -const minuteBucket = (value: Date): string => value.toISOString().slice(0, 16); -const dayBucket = (value: Date): string => value.toISOString().slice(0, 10); - -const minuteMillis = 60_000; - -/** - * Milliseconds left until the next UTC minute boundary. UTC minutes are aligned - * to the epoch, so this is exact modular arithmetic over the instant. - */ -const millisecondsUntilNextMinute = (value: Date): number => { - const remainder = value.getTime() % minuteMillis; - if (remainder === 0) return minuteMillis; - return minuteMillis - remainder; -}; - -/** JSON text of an ingest envelope as stored on the processed record. */ -const encodeEnvelopeJson = Schema.encodeSync(Schema.UnknownFromJsonString); - -/** Best-effort human text for an unknown queue/driver error. */ -const errorCauseText = (error: unknown): string => { - if (typeof error === "object" && error !== null && "cause" in error) { - return String(error.cause); - } - return String(error); -}; - -const makePolicyCounterStoreLive = Layer.effect( - PolicyCounterStore, - Effect.gen(function* () { - const store = yield* KeyValueStore; - const runtime = yield* PlatformRuntime; - const increment = (key: string, ttlMillis: number) => - store.increment("analytics-policy", key, { ttlMillis }).pipe( - Effect.provideService(PlatformRuntime, runtime), - Effect.mapError( - (error) => - new PolicyStoreError({ - cause: error.cause, - message: "policy counter increment failed", - }), - ), - ); - return PolicyCounterStore.of({ - checkEventQuota: ({ now, projectId, quota }) => { - if (typeof quota !== "number" || quota < 1) return Effect.succeed(true); - return increment(`events:${projectId}:${dayBucket(now)}`, 172_800_000).pipe( - Effect.map((count) => count <= quota), - ); - }, - checkRequestLimit: ({ now, projectId, requestsPerMinute }) => { - if (typeof requestsPerMinute !== "number" || requestsPerMinute < 1) { - return Effect.succeed({ allowed: true }); - } - return increment(`requests:${projectId}:${minuteBucket(now)}`, 120_000).pipe( - Effect.map((count) => { - if (count <= requestsPerMinute) return { allowed: true }; - return { allowed: false, retryAfterMs: millisecondsUntilNextMinute(now) }; - }), - ); - }, - }); - }), -); - -const makeCaptureIngressLive = Layer.effect( - CaptureIngress, - Effect.gen(function* () { - const queues = yield* QueueDriver; - const runtime = yield* PlatformRuntime; - const dlq = yield* AnalyticsIngestDlqService; - const producer = queues.producer(analyticsQueueName, AnalyticsIngestQueueMessage); - - const enqueueBatch = (events: ReadonlyArray) => - Effect.gen(function* () { - const publishable: AnalyticsIngestQueueMessageType[] = []; - for (const event of events) { - if ( - event.routeClass !== "main" && - event.routeClass !== "overflow" && - event.routeClass !== "historical" - ) { - yield* dlq.recordFailure({ - attemptCount: 0, - captureId: event.envelope.captureId, - distinctId: event.envelope.distinctId, - failureClass: "unsupported_route", - failureMessage: `route '${event.routeClass}' is not supported by queue ingest`, - payloadJson: event.envelope, - projectId: event.envelope.projectId, - routeClass: event.routeClass, - sourceSequence: 0, - sourceShard: "capture-ingress", - }); - continue; - } - publishable.push({ envelope: event.envelope, lane: event.routeClass }); - } - if (publishable.length > 0) { - yield* producer.publishBatch(publishable).pipe( - Effect.provideService(PlatformRuntime, runtime), - ); - } - }).pipe( - Effect.mapError( - (error) => - new CaptureIngressError({ - cause: errorCauseText(error), - message: "failed to enqueue captured analytics events", - }), - ), - ); - - return CaptureIngress.of({ enqueueBatch }); - }), -); - -/** - * Builds the process-wide platform primitives — queue, key-value store, cron - * scheduler, and runtime marker — plus the policy counter and capture services - * layered on top of them. - */ -export const makeSelfhostAnalyticsRuntimeLive = (config: SelfhostRuntimeConfig) => { - const platform = makeSelfhostPlatformLive(config); +/** Builds the synchronous PostgreSQL analytics services for Community self-host. */ +export const makeSelfhostAnalyticsRuntimeLive = ( + config: SelfhostRuntimeConfig, +): Layer.Layer => { const database = Db.layer(config.database); - const dlq = AnalyticsIngestDlqService.layer.pipe(Layer.provide(database)); - const policy = makePolicyCounterStoreLive.pipe(Layer.provide(platform)); - const ingress = makeCaptureIngressLive.pipe( - Layer.provide(dlq), - Layer.provide(database), - Layer.provide(platform), - ); + const store = AnalyticsEventStore.layer.pipe(Layer.provide(database)); const capture = EventCaptureService.layer.pipe( - Layer.provide(policy), - Layer.provide(ingress), + Layer.provide(store), Layer.provide(database), ); - const dispatch = AnalyticsDispatchService.layer.pipe(Layer.provide(ingress)); - return Layer.mergeAll(platform, capture, dispatch); + const dispatch = AnalyticsDispatchService.layer.pipe(Layer.provide(store)); + return Layer.mergeAll(store, capture, dispatch); }; - -/** Runs the analytics ingest and dead-letter consumers until their scope closes. */ -export const runSelfhostAnalyticsConsumers = ( - config: SelfhostRuntimeConfig, - clickhouse?: Layer.Layer, -) => - Effect.gen(function* () { - const queues = yield* QueueDriver; - const database = Db.layer(config.database); - const dlq = AnalyticsIngestDlqService.layer.pipe(Layer.provide(database)); - const processor = EventProcessorService.layer.pipe( - Layer.provide( - DlqProducer.dbLive.pipe( - Layer.provide(dlq), - Layer.provide(database), - ), - ), - Layer.provide( - PersonIdentityService.layer.pipe( - Layer.provide(IdentityProjectionPublisher.noop), - Layer.provide(database), - ), - ), - Layer.provide(database), - ); - const buildWriterContext = () => { - if (clickhouse === undefined) { - return Layer.build(AnalyticsWriterService.layer.pipe(Layer.provide(database))); - } - return Effect.gen(function* () { - const clickhouseContext = yield* Layer.build(clickhouse); - const client = Context.get( - clickhouseContext, - ClickhouseWebClient.ClickhouseWebClient, - ); - return yield* Layer.build( - AnalyticsWriterService.layerWithClickhouse(client).pipe( - Layer.provide(database), - ), - ); - }); - }; - const writerContext = yield* buildWriterContext(); - const analyticsWriter = Context.get(writerContext, AnalyticsWriterService); - - const consumeAnalytics = queues.consumeBatch( - analyticsQueueName, - AnalyticsIngestQueueMessage, - (messages) => - Effect.gen(function* () { - const eventProcessor = yield* EventProcessorService; - const writerMessages: AnalyticsWriterMessageType[] = []; - for (const message of messages) { - const outputs = yield* eventProcessor.processRecordToOutputs({ - capturedEvent: message.envelope, - headers: {}, - lane: message.lane, - rawValue: encodeEnvelopeJson(message.envelope), - sourceOffset: message.envelope.captureId, - sourcePartition: 0, - sourceTopic: message.envelope.routing.targetTopic, - }); - for (const processed of outputs.processedEvents) { - writerMessages.push({ - kind: "processed", - messageId: processed.processedEventId, - value: processed, - }); - } - for (const person of outputs.personEvents) { - writerMessages.push({ - kind: "person", - messageId: `${person.projectId}:${person.personId}:${person.version}`, - value: person, - }); - } - for (const identity of outputs.personIdentityEvents) { - writerMessages.push({ - kind: "person-distinct-id", - messageId: `${identity.projectId}:${identity.distinctId}:${identity.version}`, - value: identity, - }); - } - } - if (writerMessages.length > 0) { - yield* analyticsWriter.writeMessages(writerMessages); - } - }).pipe(Effect.provide(processor)), - { - batchSize: 100, - deadLetterQueue: analyticsDeadLetterQueueName, - maxRetries: 10, - }, - ); - - const consumeDeadLetters = queues.consumeBatch( - analyticsDeadLetterQueueName, - AnalyticsIngestQueueMessage, - (messages) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - yield* Effect.forEach( - messages, - (message) => - service.recordFailure({ - attemptCount: 10, - captureId: message.envelope.captureId, - distinctId: message.envelope.distinctId, - failureClass: "ingest_retry_exhausted", - failureMessage: "exhausted analytics ingest queue retries", - payloadJson: message.envelope, - projectId: message.envelope.projectId, - routeClass: message.envelope.routing.routeClass, - sourceSequence: 0, - sourceShard: analyticsDeadLetterQueueName, - }), - { discard: true }, - ); - }).pipe(Effect.provide(dlq), Effect.provide(database)), - { batchSize: 100, maxRetries: 5 }, - ); - - return yield* Effect.all([consumeAnalytics, consumeDeadLetters], { - concurrency: "unbounded", - }); - }); diff --git a/apps/backend/src/backend/Backend.ts b/apps/backend/src/backend/Backend.ts index 5b4ce15bc..0501f6d55 100644 --- a/apps/backend/src/backend/Backend.ts +++ b/apps/backend/src/backend/Backend.ts @@ -4,7 +4,6 @@ import { BackendSnapshotImageRendererStubLive, type InfraServices, } from "@voidhash/backend/BackendApp"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; import type { AuthTokenVerifier } from "@voidhash/core/services/auth/AuthTokenVerifier"; import { IdentityProvider } from "@voidhash/core/services/auth/IdentityProvider"; import { @@ -53,7 +52,6 @@ export const makeSelfhostAuthLayers = (config: SelfhostAuthConfig): SelfhostAuth export const makeBackendInfrastructureLive = ( config: SelfhostRuntimeConfig, identity: SelfhostAuthLayers["identity"], - clickhouse?: Layer.Layer, snapshotImageRenderer: Layer.Layer< SnapshotImageRenderer, never, @@ -85,6 +83,5 @@ export const makeBackendInfrastructureLive = ( // reads the same store instance merged above (memoized by reference). snapshotImageRenderer.pipe(Layer.provide(publicFileStore)), MemoryProjectSchemaCacheLive, - clickhouse ?? Layer.empty, ); }; diff --git a/apps/backend/src/backend/Background.ts b/apps/backend/src/backend/Background.ts index 7922f3b9c..1f9787bff 100644 --- a/apps/backend/src/backend/Background.ts +++ b/apps/backend/src/backend/Background.ts @@ -1,16 +1,12 @@ -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { AnalyticsJanitorService } from "@voidhash/core/services/analyticsIngest/AnalyticsJanitorService"; import { FxRateSync } from "@voidhash/core/workflows/definitions"; import { backendWorkflows } from "@voidhash/core/workflows/registry"; import { CronJob, CronScheduler } from "@voidhash/platform/CronScheduler"; import type { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; import type { WorkflowRunner } from "@voidhash/platform/WorkflowRunner"; -import { Config, Context, Effect, Layer } from "effect"; +import { Config, Effect } from "effect"; /** Builds the persisted jobs enabled by the current self-host configuration. */ -export const makeSelfhostCronJobs = ( - clickhouse?: Layer.Layer, -) => +export const makeSelfhostCronJobs = Effect.gen(function* () { const exchangeRateApiKey = (yield* Config.string("EXCHANGE_RATE_API_KEY").pipe( Config.withDefault(""), @@ -30,31 +26,14 @@ export const makeSelfhostCronJobs = ( }, ); - if (clickhouse) { - const janitorContext = yield* Layer.build( - AnalyticsJanitorService.layer.pipe(Layer.provide(clickhouse)), - ); - const janitor = Context.get(janitorContext, AnalyticsJanitorService); - jobs.push( - CronJob.define({ - expression: "*/5 * * * *", - name: "analytics-janitor", - run: () => - janitor.squash({ batchSize: 1000, safetyWindowSeconds: 120 }).pipe(Effect.asVoid), - }), - ); - } - return jobs; }); /** Runs every enabled persisted cron job until the enclosing scope closes. */ -export const runSelfhostCronJobs = ( - clickhouse?: Layer.Layer, -) => +export const runSelfhostCronJobs = Effect.gen(function* () { const scheduler = yield* CronScheduler; - const jobs = yield* makeSelfhostCronJobs(clickhouse); + const jobs = yield* makeSelfhostCronJobs; return yield* Effect.all( jobs.map((job) => scheduler.run(job, { pollIntervalMillis: 1_000 })), { concurrency: "unbounded" }, diff --git a/apps/backend/src/backend/Clickhouse.ts b/apps/backend/src/backend/Clickhouse.ts deleted file mode 100644 index a01788842..000000000 --- a/apps/backend/src/backend/Clickhouse.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { ClickhouseDbLive } from "@voidhash/clickhouse-db"; -import { analyticsEventsMigrations } from "@voidhash/clickhouse-db/analytics/migration"; -import { - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSONS_TABLE, -} from "@voidhash/clickhouse-db/analytics/schema"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { constant } from "@voidhash/lib/lang"; -import { Effect, Layer, Schedule } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import type { SelfhostClickhouseConfig } from "../config.ts"; - -const tenantTables = constant([ - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSONS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, -]); - -const queryTables = constant([ - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSONS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, -]); - -const identifierPattern = /^[A-Za-z_][A-Za-z0-9_]*$/; - -/** - * Guards a configured ClickHouse identifier before it is interpolated into DDL. - * A bad identifier is a deployment misconfiguration, not a recoverable failure, - * so it is raised as a defect exactly as the previous `throw` was. - */ -const assertIdentifier = (name: string, value: string): Effect.Effect => { - if (!identifierPattern.test(value)) { - return Effect.die(new Error(`${name} must be a ClickHouse identifier`)); - } - return Effect.succeed(value); -}; - -const makeClientLive = (config: SelfhostClickhouseConfig["readWrite"]) => - ClickhouseDbLive(config).pipe(Layer.orDie); - -/** Builds the read-write, tenant-readonly, and hardened query client layers. */ -export const makeSelfhostClickhouseLayers = (config: SelfhostClickhouseConfig) => ({ - analyticsQuery: makeClientLive(config.analyticsQuery), - readOnly: makeClientLive(config.readOnly), - readWrite: makeClientLive(config.readWrite), -}); - -const provisionSelfhostClickhouseAccess = (config: SelfhostClickhouseConfig) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - const database = yield* assertIdentifier("CLICKHOUSE_DATABASE", config.admin.database); - const adminUser = yield* assertIdentifier( - "CLICKHOUSE_ADMIN_USERNAME", - config.admin.username, - ); - const readWriteUser = yield* assertIdentifier( - "CLICKHOUSE_USERNAME", - config.readWrite.username, - ); - const readOnlyUser = yield* assertIdentifier( - "CLICKHOUSE_RO_USERNAME", - config.readOnly.username, - ); - const queryUser = yield* assertIdentifier( - "CLICKHOUSE_ANALYTICS_QUERY_USERNAME", - config.analyticsQuery.username, - ); - const readWriteRole = `${database}_app_role`; - const readOnlyRole = `${database}_ro_role`; - const queryRole = `${database}_query_role`; - - for (const [user, password] of constant([ - [readWriteUser, config.readWrite.password], - [readOnlyUser, config.readOnly.password], - [queryUser, config.analyticsQuery.password], - ])) { - yield* ch.asCommand(sql` - CREATE USER IF NOT EXISTS ${sql(user)} IDENTIFIED WITH sha256_password BY ${password} - `); - yield* ch.asCommand(sql` - ALTER USER ${sql(user)} IDENTIFIED WITH sha256_password BY ${password} - `); - } - - for (const role of [readWriteRole, readOnlyRole, queryRole]) { - yield* ch.asCommand(sql`CREATE ROLE IF NOT EXISTS ${sql(role)}`); - } - yield* ch.asCommand(sql`GRANT ALL ON ${sql(database)}.* TO ${sql(readWriteRole)}`); - for (const table of tenantTables) { - yield* ch.asCommand( - sql`GRANT SELECT ON ${sql(database)}.${sql(table)} TO ${sql(readOnlyRole)}`, - ); - } - for (const table of queryTables) { - yield* ch.asCommand( - sql`GRANT SELECT ON ${sql(database)}.${sql(table)} TO ${sql(queryRole)}`, - ); - } - yield* ch.asCommand(sql`GRANT ${sql(readWriteRole)} TO ${sql(readWriteUser)}`); - yield* ch.asCommand(sql`GRANT ${sql(readOnlyRole)} TO ${sql(readOnlyUser)}`); - yield* ch.asCommand(sql`GRANT ${sql(queryRole)} TO ${sql(queryUser)}`); - yield* ch.asCommand( - sql`ALTER USER ${sql(readWriteUser)} DEFAULT ROLE ${sql(readWriteRole)}`, - ); - yield* ch.asCommand(sql`ALTER USER ${sql(readOnlyUser)} DEFAULT ROLE ${sql(readOnlyRole)}`); - yield* ch.asCommand(sql`ALTER USER ${sql(queryUser)} DEFAULT ROLE ${sql(queryRole)}`); - yield* ch.asCommand(sql` - ALTER USER ${sql(readOnlyUser)} SETTINGS - readonly = 1, - SQL_organization_id = '' CHANGEABLE_IN_READONLY - `); - yield* ch.asCommand(sql` - ALTER USER ${sql(queryUser)} SETTINGS - readonly = 1, - allow_ddl = 0, - allow_introspection_functions = 0, - max_execution_time = 30, - max_result_rows = 100000 - `); - - for (const table of tenantTables) { - const tenantPolicy = `${database}_${table}_tenant`; - const unrestrictedPolicy = `${database}_${table}_unrestricted`; - yield* ch.asCommand(sql` - CREATE ROW POLICY OR REPLACE ${sql(tenantPolicy)} - ON ${sql(database)}.${sql(table)} - FOR SELECT - USING getSetting('SQL_organization_id') != '' - AND organization_id = getSetting('SQL_organization_id') - TO ${sql(readOnlyRole)} - `); - yield* ch.asCommand(sql` - CREATE ROW POLICY OR REPLACE ${sql(unrestrictedPolicy)} - ON ${sql(database)}.${sql(table)} - FOR SELECT USING 1 - TO ${sql(readWriteRole)}, ${sql(queryRole)}, ${sql(adminUser)} - `); - } - }); - -/** Applies analytics schema migrations and reconciles local ClickHouse access roles. */ -export const migrateSelfhostClickhouse = (config: SelfhostClickhouseConfig | undefined) => { - if (config === undefined) return Effect.void; - const admin = ClickhouseDbLive(config.admin, [analyticsEventsMigrations]); - return provisionSelfhostClickhouseAccess(config).pipe( - Effect.provide(admin), - Effect.scoped, - Effect.retry({ schedule: Schedule.spaced("1 second"), times: 30 }), - ); -}; diff --git a/apps/backend/src/config.ts b/apps/backend/src/config.ts index 73ec61e25..6c5a444ed 100644 --- a/apps/backend/src/config.ts +++ b/apps/backend/src/config.ts @@ -73,14 +73,6 @@ export const validateSelfhostSecurityConfig = (): SelfhostMode => { const unsafeSettings: Array = [...standaloneAuthConfigIssues()]; const requiredSecrets = ["DATABASE_PASSWORD", "MIMIC_ROOT_PASSWORD", "S3_SECRET_ACCESS_KEY"]; - if (process.env.CLICKHOUSE_URL?.trim()) { - requiredSecrets.push( - "CLICKHOUSE_ADMIN_PASSWORD", - "CLICKHOUSE_PASSWORD", - "CLICKHOUSE_RO_PASSWORD", - "CLICKHOUSE_ANALYTICS_QUERY_PASSWORD", - ); - } for (const name of requiredSecrets) { if (isPlaceholderSecret(process.env[name])) unsafeSettings.push(name); } @@ -129,22 +121,6 @@ export const getSelfhostAuthConfig = (): SelfhostAuthConfig => { }; }; -/** A named ClickHouse connection used by the self-host analytics runtime. */ -export interface SelfhostClickhouseConnection { - readonly database: string; - readonly password: string; - readonly url: string; - readonly username: string; -} - -/** ClickHouse administrative and least-privilege runtime connections. */ -export interface SelfhostClickhouseConfig { - readonly admin: SelfhostClickhouseConnection; - readonly analyticsQuery: SelfhostClickhouseConnection; - readonly readOnly: SelfhostClickhouseConnection; - readonly readWrite: SelfhostClickhouseConnection; -} - /** BYO-provider configuration for durable self-hosted agent sessions. */ export interface SelfhostAgentConfig { readonly provider: string; @@ -160,7 +136,6 @@ export interface SelfhostAgentConfig { export interface SelfhostRuntimeConfig { readonly agent: SelfhostAgentConfig; readonly auth: SelfhostAuthConfig; - readonly clickhouse?: SelfhostClickhouseConfig; readonly componentCompilerUrl: string; readonly database: DbConfig; readonly host: string; @@ -178,37 +153,6 @@ export interface SelfhostRuntimeConfig { readonly artifactObjectStore: S3ObjectStoreConfig; } -/** Reads optional ClickHouse configuration, returning undefined when analytics is disabled. */ -export const getSelfhostClickhouseConfig = (): SelfhostClickhouseConfig | undefined => { - const url = process.env.CLICKHOUSE_URL?.trim(); - if (!url) return undefined; - const database = process.env.CLICKHOUSE_DATABASE?.trim() || "voidhash"; - const connection = (username: string, password: string): SelfhostClickhouseConnection => ({ - database, - password, - url, - username, - }); - return { - admin: connection( - process.env.CLICKHOUSE_ADMIN_USERNAME?.trim() || "voidhash_admin", - process.env.CLICKHOUSE_ADMIN_PASSWORD ?? "password", - ), - analyticsQuery: connection( - process.env.CLICKHOUSE_ANALYTICS_QUERY_USERNAME?.trim() || "voidhash_query", - process.env.CLICKHOUSE_ANALYTICS_QUERY_PASSWORD ?? "password", - ), - readOnly: connection( - process.env.CLICKHOUSE_RO_USERNAME?.trim() || "voidhash_ro", - process.env.CLICKHOUSE_RO_PASSWORD ?? "password", - ), - readWrite: connection( - process.env.CLICKHOUSE_USERNAME?.trim() || "voidhash_app", - process.env.CLICKHOUSE_PASSWORD ?? "password", - ), - }; -}; - /** Reads the shared application database connection from environment variables. */ export const getSelfhostDatabaseConfig = (): DbConfig => ({ databaseName: process.env.DATABASE_NAME?.trim() || "voidhash", @@ -333,14 +277,6 @@ const agentOpenaiBaseUrl = (): { readonly openaiBaseUrl?: string } => { return { openaiBaseUrl }; }; -/** Omits the ClickHouse block entirely when analytics is disabled. */ -const optionalClickhouse = ( - clickhouse: SelfhostClickhouseConfig | undefined, -): { readonly clickhouse?: SelfhostClickhouseConfig } => { - if (clickhouse === undefined) return {}; - return { clickhouse }; -}; - /** Reads and validates the complete single-process runtime configuration. */ export const getSelfhostRuntimeConfig = (): SelfhostRuntimeConfig => { validateSelfhostSecurityConfig(); @@ -356,7 +292,6 @@ export const getSelfhostRuntimeConfig = (): SelfhostRuntimeConfig => { region, secretAccessKey, }; - const clickhouse = getSelfhostClickhouseConfig(); const openaiApiKey = process.env.OPENAI_API_KEY?.trim(); const anthropicApiKey = process.env.ANTHROPIC_API_KEY?.trim(); const { modelId: defaultModelId, provider: defaultProvider } = defaultAgentModel(openaiApiKey); @@ -376,7 +311,6 @@ export const getSelfhostRuntimeConfig = (): SelfhostRuntimeConfig => { }, auth: getSelfhostAuthConfig(), database: getSelfhostDatabaseConfig(), - ...optionalClickhouse(clickhouse), componentCompilerUrl: process.env.COMPONENT_COMPILER_URL?.trim() || "http://127.0.0.1:5002", host: process.env.HOST?.trim() || "0.0.0.0", mailer: getSelfhostSmtpConfig(), diff --git a/apps/backend/src/migrations.ts b/apps/backend/src/migrations.ts index b0be83062..14c120885 100644 --- a/apps/backend/src/migrations.ts +++ b/apps/backend/src/migrations.ts @@ -2,10 +2,8 @@ import { runAppDatabaseMigrations } from "@voidhash/db/migrations"; import { PgClusterDurableEntityLive } from "@voidhash/platform-selfhost/ClusterDurableEntity"; import { Effect, Layer } from "effect"; -import { migrateSelfhostClickhouse } from "./backend/Clickhouse.ts"; import { selfhostPlatformPostgres } from "./backend/PlatformProfile.ts"; import { - getSelfhostClickhouseConfig, getSelfhostMigrationDatabaseConfig, getSelfhostPlatformDatabaseConfig, validateSelfhostSecurityConfig, @@ -24,8 +22,7 @@ export interface SelfhostMigrationOptions { /** * Applies every migration the self-host runtime needs: the application schema, - * the mimic document control tables, and — when analytics is configured — the - * ClickHouse schema. + * the mimic document control tables, and platform persistence tables. */ export const runSelfhostMigrations = (options: SelfhostMigrationOptions = {}) => Effect.gen(function* () { @@ -47,6 +44,5 @@ export const runSelfhostMigrations = (options: SelfhostMigrationOptions = {}) => yield* Layer.build( makeMimicNodeHostLive(mimicConfig, PgClusterDurableEntityLive(platform)), ); - yield* migrateSelfhostClickhouse(getSelfhostClickhouseConfig()); yield* Effect.logInfo("Self-host database migrations are ready", { applied, skipped }); }); diff --git a/apps/backend/src/server.ts b/apps/backend/src/server.ts index 32e156a70..cfdc38b9e 100644 --- a/apps/backend/src/server.ts +++ b/apps/backend/src/server.ts @@ -31,13 +31,9 @@ import { HttpApiBuilder } from "effect/unstable/httpapi"; import type * as Rpc from "effect/unstable/rpc/Rpc"; import { EventCaptureGroupLive } from "@voidhash/backend/routes/event-capture"; -import { - makeSelfhostAnalyticsRuntimeLive, - runSelfhostAnalyticsConsumers, -} from "./backend/Analytics.ts"; +import { makeSelfhostAnalyticsRuntimeLive } from "./backend/Analytics.ts"; import { runSelfhostCronJobs } from "./backend/Background.ts"; import { makeBackendInfrastructureLive, makeSelfhostAuthLayers } from "./backend/Backend.ts"; -import { makeSelfhostClickhouseLayers } from "./backend/Clickhouse.ts"; import { runSelfhostPushDeliveryConsumers, SelfhostPushDeliveryDispatchLive, @@ -79,11 +75,6 @@ const optionalEnv = (name: string): Effect.Effect => const optionalTrimmedEnv = (name: string): Effect.Effect => optionalEnv(name).pipe(Effect.map((value) => value?.trim())); -const makeClickhouseLayers = (config: SelfhostRuntimeConfig) => { - if (!config.clickhouse) return undefined; - return makeSelfhostClickhouseLayers(config.clickhouse); -}; - const makeChromiumConfig = ( executablePath: string | undefined, disableSandbox: boolean, @@ -190,7 +181,6 @@ export const runSelfhostServer = < yield* Effect.logInfo( `Identity provider: standalone (root user ${config.auth.rootUsername})`, ); - const clickhouse = makeClickhouseLayers(config); const chromiumExecutablePath = yield* optionalTrimmedEnv("CHROMIUM_EXECUTABLE_PATH"); const chromiumDisableSandbox = yield* optionalEnv("CHROMIUM_DISABLE_SANDBOX"); const chromiumConfig = makeChromiumConfig( @@ -201,7 +191,6 @@ export const runSelfhostServer = < makeBackendInfrastructureLive( config, authLayers.identity, - clickhouse?.readOnly, makeSnapshotImageRenderer(chromiumConfig), ), options.identityDirectory ?? Layer.empty, @@ -209,12 +198,18 @@ export const runSelfhostServer = < const authContext = yield* Layer.build(authLayers.authTokenVerifier); const authTokenVerifier = Context.get(authContext, AuthTokenVerifier); const rpcExtension = options.rpcExtension({ authTokenVerifier, config }); - const workflowRuntime = Layer.merge(platform.workflowRunner, platform.runtime); + const platformRuntime = Layer.mergeAll( + platform.workflowRunner, + platform.runtime, + platform.queue, + platform.keyValueStore, + platform.cronScheduler, + ); const analyticsRuntime = makeSelfhostAnalyticsRuntimeLive(config); const runtimeContext = yield* Layer.build( Layer.mergeAll( infrastructure, - workflowRuntime, + platformRuntime, analyticsRuntime, SmtpMailerLive(config.mailer), ), @@ -252,16 +247,11 @@ export const runSelfhostServer = < (registration) => registration.register(workflowInfra), { discard: true }, ).pipe(Effect.provide(runtimeContext), Effect.orDie); - yield* Effect.forkScoped( - runSelfhostAnalyticsConsumers(config, clickhouse?.readWrite).pipe( - Effect.provide(runtimeContext), - ), - ); yield* Effect.forkScoped( runSelfhostPushDeliveryConsumers(config).pipe(Effect.provide(runtimeContext)), ); yield* Effect.forkScoped( - runSelfhostCronJobs(clickhouse?.readWrite).pipe(Effect.provide(runtimeContext)), + runSelfhostCronJobs.pipe(Effect.provide(runtimeContext)), ); if (chromiumConfig !== undefined) { const thumbnailContext = yield* Layer.build( @@ -285,7 +275,6 @@ export const runSelfhostServer = < features: options.features, rpcExtension, infrastructure, - analyticsQueryClient: clickhouse?.analyticsQuery, pushDeliveryDispatch, routeExtension: options.routeExtension, mcpOAuth: options.mcpOAuth, diff --git a/apps/backend/tests/Analytics.integration.test.ts b/apps/backend/tests/Analytics.integration.test.ts index 6a298d92b..2d9c33349 100644 --- a/apps/backend/tests/Analytics.integration.test.ts +++ b/apps/backend/tests/Analytics.integration.test.ts @@ -1,25 +1,15 @@ import { EventCaptureService } from "@voidhash/core/services/analyticsIngest/EventCaptureService"; import { generateId } from "@voidhash/core/utils/generate-id"; -import { - Db, - apiKeys, - eq, - personIdentities, - persons, - projects, - sql, -} from "@voidhash/db"; -import { Clock, DateTime, Effect } from "effect"; +import { Db, analyticsEvents, apiKeys, eq, projects } from "@voidhash/db"; +import { DateTime, Effect } from "effect"; import { describe, expect, it } from "vitest"; -import { - makeSelfhostAnalyticsRuntimeLive, - runSelfhostAnalyticsConsumers, -} from "../src/backend/Analytics.ts"; +import { makeSelfhostAnalyticsRuntimeLive } from "../src/backend/Analytics.ts"; +import { makeSelfhostPlatformLive } from "../src/backend/PlatformProfile.ts"; import { getSelfhostRuntimeConfig } from "../src/config.ts"; -describe("self-host analytics queue", () => { - it("captures, processes, and acknowledges an event without ClickHouse", () => +describe("self-host PostgreSQL analytics", () => { + it("persists an allowed event before capture returns", () => Effect.runPromise( Effect.gen(function* () { const config = getSelfhostRuntimeConfig(); @@ -27,87 +17,65 @@ describe("self-host analytics queue", () => { const projectId = `project_capture_${suffix}`; const token = `vh_pk_capture_${suffix.replaceAll("-", "")}`; const database = Db.layer(config.database); - const program = Effect.scoped( - Effect.gen(function* () { - const db = yield* Db; - const now = yield* DateTime.nowAsDate; - yield* db.insert(projects).values({ - id: projectId, - name: "Capture integration", - organizationId: `organization_${suffix}`, - slug: `capture-${suffix}`, - }); - yield* db.insert(apiKeys).values({ - end: token.slice(-4), - id: `apiKey_capture_${suffix}`, - isPublic: true, - key: token, - name: "Capture integration", - prefix: "vh_pk_", - projectId, - }); - yield* Effect.forkScoped(runSelfhostAnalyticsConsumers(config)); - const capture = yield* EventCaptureService; - const result = yield* capture.captureEvents({ - events: [ - { - context: {}, - distinct_id: `person_${suffix}`, - event: "selfhost_integration", - properties: { plan: "pro" }, - uuid: `event_${suffix}`, - }, - ], - request: { - headers: {}, - path: "/i/v1/capture", - receivedAt: now, - requestId: `request_${suffix}`, - sentAt: now, - token, + const program = Effect.gen(function* () { + const db = yield* Db; + const now = yield* DateTime.nowAsDate; + yield* db.insert(projects).values({ + id: projectId, + name: "Capture integration", + organizationId: `organization_${suffix}`, + slug: `capture-${suffix}`, + }); + yield* db.insert(apiKeys).values({ + end: token.slice(-4), + id: `apiKey_capture_${suffix}`, + isPublic: true, + key: token, + name: "Capture integration", + prefix: "vh_pk_", + projectId, + }); + + const capture = yield* EventCaptureService; + const result = yield* capture.captureEvents({ + events: [ + { + context: {}, + distinct_id: `person_${suffix}`, + event: "$app_opened", + properties: { plan: "pro" }, + uuid: `event_${suffix}`, }, - }); - expect(result).toEqual({ accepted: 1, rejected: 0 }); + ], + request: { + headers: {}, + path: "/i/v1/capture", + receivedAt: now, + requestId: `request_${suffix}`, + sentAt: now, + token, + }, + }); - const deadline = (yield* Clock.currentTimeMillis) + 10_000; - while ((yield* Clock.currentTimeMillis) < deadline) { - const rows = yield* db.query.persons.findMany({ where: { projectId } }); - if (rows.length > 0) return rows.length; - yield* Effect.sleep("25 millis"); - } - return yield* Effect.die("analytics queue did not process the captured event"); - }).pipe( - Effect.provide(database), - Effect.provide(makeSelfhostAnalyticsRuntimeLive(config)), - ), + expect(result).toEqual({ accepted: 1, rejected: 0 }); + const rows = yield* db.query.analyticsEvents.findMany({ where: { projectId } }); + expect(rows).toHaveLength(1); + expect(rows[0]?.eventName).toBe("$app_opened"); + }).pipe( + Effect.provide(makeSelfhostAnalyticsRuntimeLive(config)), + Effect.provide(makeSelfhostPlatformLive(config)), + Effect.provide(database), ); const cleanup = Effect.gen(function* () { const db = yield* Db; + yield* db.delete(analyticsEvents).where(eq(analyticsEvents.projectId, projectId)); yield* db.delete(apiKeys).where(eq(apiKeys.projectId, projectId)); - yield* db.delete(personIdentities).where(eq(personIdentities.projectId, projectId)); - yield* db.delete(persons).where(eq(persons.projectId, projectId)); yield* db.delete(projects).where(eq(projects.id, projectId)); }).pipe(Effect.provide(database), Effect.orDie); - const cleanupQueue = Effect.gen(function* () { - const db = yield* Db; - // The cluster queue driver hands the store a JSON string, which the - // store then JSON-encodes into `element`, so the body is doubly - // encoded: unwrap the outer JSON scalar before reading its fields. - yield* db.execute(sql` - DELETE FROM effect_queue - WHERE (element::jsonb #>> '{}')::jsonb -> 'envelope' ->> 'projectId' = ${projectId} - `); - }).pipe(Effect.provide(Db.layer(config.platformDatabase)), Effect.orDie); - - const count = yield* program.pipe( - Effect.ensuring(cleanup), - Effect.ensuring(cleanupQueue), - ); - - expect(count).toBe(1); + yield* program.pipe(Effect.ensuring(cleanup)); }), )); }); diff --git a/apps/backend/tests/BackendAdapters.test.ts b/apps/backend/tests/BackendAdapters.test.ts index b9d8582ff..834731390 100644 --- a/apps/backend/tests/BackendAdapters.test.ts +++ b/apps/backend/tests/BackendAdapters.test.ts @@ -32,7 +32,6 @@ describe("self-host runtime configuration", () => { configTest("uses local development defaults", () => { delete process.env.NODE_ENV; delete process.env.ANTHROPIC_API_KEY; - delete process.env.CLICKHOUSE_URL; delete process.env.PUBLIC_BASE_URL; delete process.env.OPENAI_API_KEY; delete process.env.S3_ENDPOINT; @@ -53,7 +52,6 @@ describe("self-host runtime configuration", () => { const config = getSelfhostRuntimeConfig(); - expect(config.clickhouse).toBeUndefined(); expect(config.agent).toMatchObject({ modelId: "claude-sonnet-4-6", provider: "anthropic", @@ -188,21 +186,6 @@ describe("self-host runtime configuration", () => { }); }); - configTest("enables ClickHouse only when its URL is configured", () => { - process.env.CLICKHOUSE_URL = "http://clickhouse:8123"; - process.env.CLICKHOUSE_DATABASE = "analytics"; - delete process.env.CLICKHOUSE_ADMIN_USERNAME; - delete process.env.CLICKHOUSE_ANALYTICS_QUERY_USERNAME; - delete process.env.CLICKHOUSE_RO_USERNAME; - delete process.env.CLICKHOUSE_USERNAME; - - expect(getSelfhostRuntimeConfig().clickhouse).toMatchObject({ - admin: { database: "analytics", username: "voidhash_admin" }, - analyticsQuery: { username: "voidhash_query" }, - readOnly: { username: "voidhash_ro" }, - readWrite: { username: "voidhash_app" }, - }); - }); }); describe("memory project schema cache", () => { diff --git a/apps/backend/tests/Background.integration.test.ts b/apps/backend/tests/Background.integration.test.ts index 58d78c5e6..0f402e462 100644 --- a/apps/backend/tests/Background.integration.test.ts +++ b/apps/backend/tests/Background.integration.test.ts @@ -7,8 +7,8 @@ import { constant } from "@voidhash/lib/lang"; import { Clock, DateTime, Effect, Layer } from "effect"; import { describe, expect, it } from "vitest"; -import { makeSelfhostAnalyticsRuntimeLive } from "../src/backend/Analytics.ts"; import { makeSelfhostCronJobs } from "../src/backend/Background.ts"; +import { makeSelfhostPlatformLive } from "../src/backend/PlatformProfile.ts"; import { getSelfhostRuntimeConfig } from "../src/config.ts"; const requiredJobNames = constant([ @@ -59,12 +59,13 @@ describe("self-host scheduled jobs", () => { it("registers the required background jobs and executes them through the scheduler", () => Effect.runPromise( Effect.gen(function* () { + const config = getSelfhostRuntimeConfig(); const testRunner = TestWorkflowRunner.make(); const outcome = yield* Effect.scoped( Effect.gen(function* () { const jobs: ReadonlyArray> = - yield* makeSelfhostCronJobs(); + yield* makeSelfhostCronJobs; const registered = jobs.map((job) => job.name); const executions: Record = {}; for (const name of requiredJobNames) { @@ -75,7 +76,7 @@ describe("self-host scheduled jobs", () => { return { executions, registered }; }).pipe( Effect.provide(Layer.succeed(WorkflowRunner, testRunner)), - Effect.provide(makeSelfhostAnalyticsRuntimeLive(getSelfhostRuntimeConfig())), + Effect.provide(makeSelfhostPlatformLive(config)), ), ); diff --git a/apps/backend/tests/Clickhouse.integration.test.ts b/apps/backend/tests/Clickhouse.integration.test.ts deleted file mode 100644 index 4d48bbfe9..000000000 --- a/apps/backend/tests/Clickhouse.integration.test.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSONS_TABLE, -} from "@voidhash/clickhouse-db/analytics/schema"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { EventCaptureService } from "@voidhash/core/services/analyticsIngest/EventCaptureService"; -import { generateId } from "@voidhash/core/utils/generate-id"; -import { - apiKeys, - Db, - eq, - personIdentities, - persons, - projects, - sql as pgSql, -} from "@voidhash/db"; -import { constant } from "@voidhash/lib/lang"; -import { Clock, Context, Data, DateTime, Effect, Layer } from "effect"; -import { describe, expect, it } from "vitest"; - -import { - makeSelfhostAnalyticsRuntimeLive, - runSelfhostAnalyticsConsumers, -} from "../src/backend/Analytics.ts"; -import { - makeSelfhostClickhouseLayers, - migrateSelfhostClickhouse, -} from "../src/backend/Clickhouse.ts"; -import { getSelfhostRuntimeConfig } from "../src/config.ts"; - -const analyticsTables = constant([ - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSONS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, -]); - -class MissingClickhouseConfigError extends Data.TaggedError("MissingClickhouseConfigError")<{ - readonly message: string; -}> {} - -const countEvents = ( - layer: Layer.Layer, - projectId: string, - organizationId?: string, -) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const query = ch<{ readonly total: string }>` - SELECT count() AS total - FROM ${ch.literal(CLICKHOUSE_EVENTS_TABLE)} - WHERE project_id = ${ch.param("String", projectId)} - `; - if (!organizationId) { - const rows = yield* query; - return Number(rows[0]?.total ?? 0); - } - const rows = yield* ch.withClickhouseSettings(query, { - SQL_organization_id: organizationId, - }); - return Number(rows[0]?.total ?? 0); - }).pipe(Effect.provide(layer), Effect.scoped); - -describe("self-host ClickHouse analytics", () => { - it("writes captured events and enforces the runtime access split", () => - Effect.runPromise( - Effect.gen(function* () { - const config = getSelfhostRuntimeConfig(); - const clickhouseConfig = config.clickhouse; - if (!clickhouseConfig) { - return yield* new MissingClickhouseConfigError({ - message: "CLICKHOUSE_URL is required for this test", - }); - } - yield* migrateSelfhostClickhouse(clickhouseConfig); - const clickhouse = makeSelfhostClickhouseLayers(clickhouseConfig); - const database = Db.layer(config.database); - const suffix = generateId("test"); - const projectId = `project_clickhouse_${suffix}`; - const organizationId = `organization_clickhouse_${suffix}`; - const token = `vh_pk_clickhouse_${suffix.replaceAll("-", "")}`; - - const teardown = Effect.gen(function* () { - yield* Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - yield* Effect.forEach( - analyticsTables, - (table) => - ch.asCommand(ch` - ALTER TABLE ${ch(table)} DELETE - WHERE project_id = ${projectId} - `), - { discard: true }, - ); - }).pipe(Effect.provide(clickhouse.readWrite), Effect.scoped); - yield* Effect.gen(function* () { - const db = yield* Db; - yield* db.delete(apiKeys).where(eq(apiKeys.projectId, projectId)); - yield* db.delete(personIdentities).where(eq(personIdentities.projectId, projectId)); - yield* db.delete(persons).where(eq(persons.projectId, projectId)); - yield* db.delete(projects).where(eq(projects.id, projectId)); - }).pipe(Effect.provide(database)); - yield* Effect.gen(function* () { - const db = yield* Db; - // The cluster queue driver hands the store a JSON string, which the - // store then JSON-encodes into `element`, so the body is doubly - // encoded: unwrap the outer JSON scalar before reading its fields. - yield* db.execute(pgSql` - DELETE FROM effect_queue - WHERE (element::jsonb #>> '{}')::jsonb -> 'envelope' ->> 'projectId' = ${projectId} - `); - }).pipe(Effect.provide(Db.layer(config.platformDatabase))); - }).pipe(Effect.orDie); - - return yield* Effect.gen(function* () { - const written = yield* Effect.scoped( - Effect.gen(function* () { - const db = yield* Db; - yield* db.insert(projects).values({ - id: projectId, - name: "ClickHouse integration", - organizationId, - slug: `clickhouse-${suffix}`, - }); - yield* db.insert(apiKeys).values({ - end: token.slice(-4), - id: `apiKey_clickhouse_${suffix}`, - isPublic: true, - key: token, - name: "ClickHouse integration", - prefix: "vh_pk_", - projectId, - }); - - yield* Effect.forkScoped( - runSelfhostAnalyticsConsumers(config, clickhouse.readWrite), - ); - const capture = yield* EventCaptureService; - const now = yield* DateTime.nowAsDate; - yield* capture.captureEvents({ - events: [ - { - context: {}, - distinct_id: `person_${suffix}`, - event: "selfhost_clickhouse_integration", - properties: { plan: "pro" }, - uuid: `event_${suffix}`, - }, - ], - request: { - headers: {}, - path: "/i/v1/capture", - receivedAt: now, - requestId: `request_${suffix}`, - sentAt: now, - token, - }, - }); - - const readWriteContext = yield* Layer.build(clickhouse.readWrite); - const ch = Context.get( - readWriteContext, - ClickhouseWebClient.ClickhouseWebClient, - ); - const deadline = (yield* Clock.currentTimeMillis) + 10_000; - while ((yield* Clock.currentTimeMillis) < deadline) { - const rows = yield* ch<{ readonly total: string }>` - SELECT count() AS total - FROM ${ch.literal(CLICKHOUSE_EVENTS_TABLE)} - WHERE project_id = ${ch.param("String", projectId)} - `; - if (Number(rows[0]?.total ?? 0) > 0) return Number(rows[0]?.total); - yield* Effect.sleep("25 millis"); - } - return yield* Effect.die("analytics event did not land in ClickHouse"); - }).pipe( - Effect.provide(database), - Effect.provide(makeSelfhostAnalyticsRuntimeLive(config)), - Effect.provide(clickhouse.readOnly), - ), - ); - - expect(written).toBe(1); - expect(yield* countEvents(clickhouse.readOnly, projectId, organizationId)).toBe(1); - expect(yield* countEvents(clickhouse.readOnly, projectId, "another-organization")).toBe( - 0, - ); - expect(yield* countEvents(clickhouse.analyticsQuery, projectId)).toBe(1); - }).pipe(Effect.ensuring(teardown)); - }), - ), 30_000); -}); diff --git a/apps/backend/tests/Push.integration.test.ts b/apps/backend/tests/Push.integration.test.ts index 647808257..d3b05eb98 100644 --- a/apps/backend/tests/Push.integration.test.ts +++ b/apps/backend/tests/Push.integration.test.ts @@ -4,7 +4,7 @@ import { Db, sql } from "@voidhash/db"; import { Clock, Context, Effect, Layer, Predicate } from "effect"; import { describe, expect, it } from "vitest"; -import { makeSelfhostAnalyticsRuntimeLive } from "../src/backend/Analytics.ts"; +import { makeSelfhostPlatformLive } from "../src/backend/PlatformProfile.ts"; import { runSelfhostPushDeliveryConsumers, SelfhostPushDeliveryDispatchLive, @@ -62,7 +62,7 @@ describe("self-host push-delivery queue", () => { yield* Effect.sleep("25 millis"); } return 1; - }).pipe(Effect.provide(makeSelfhostAnalyticsRuntimeLive(config))), + }).pipe(Effect.provide(makeSelfhostPlatformLive(config))), ); expect(remaining).toBe(0); diff --git a/apps/backend/tests/SecurityConfig.test.ts b/apps/backend/tests/SecurityConfig.test.ts index 9d277bd4a..5f8d6e1a5 100644 --- a/apps/backend/tests/SecurityConfig.test.ts +++ b/apps/backend/tests/SecurityConfig.test.ts @@ -4,7 +4,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { validateSelfhostSecurityConfig } from "../src/config.ts"; const validProductionEnvironment = constant({ - CLICKHOUSE_URL: "", DATABASE_PASSWORD: "database-secret", MIMIC_PUBLIC_BASE_URL: "https://mimic.example.test", MIMIC_ROOT_PASSWORD: "mimic-secret", @@ -75,16 +74,6 @@ describe("validateSelfhostSecurityConfig", () => { expect(() => validateSelfhostSecurityConfig()).toThrow("VOIDHASH_ROOT_PASSWORD"); }); - it("requires every enabled ClickHouse role to have a non-example password", () => { - stubEnvironment(validProductionEnvironment); - vi.stubEnv("CLICKHOUSE_URL", "http://clickhouse:8123"); - vi.stubEnv("CLICKHOUSE_ADMIN_PASSWORD", "configured"); - vi.stubEnv("CLICKHOUSE_PASSWORD", "configured"); - vi.stubEnv("CLICKHOUSE_RO_PASSWORD", "configured"); - vi.stubEnv("CLICKHOUSE_ANALYTICS_QUERY_PASSWORD", "password"); - expect(() => validateSelfhostSecurityConfig()).toThrow("CLICKHOUSE_ANALYTICS_QUERY_PASSWORD"); - }); - it.each([ "PUBLIC_BASE_URL", "PUBLIC_FILES_BASE_URL", diff --git a/apps/backend/tests/ThumbnailQueue.integration.test.ts b/apps/backend/tests/ThumbnailQueue.integration.test.ts index a19c079ad..191dc5012 100644 --- a/apps/backend/tests/ThumbnailQueue.integration.test.ts +++ b/apps/backend/tests/ThumbnailQueue.integration.test.ts @@ -4,7 +4,7 @@ import { Db, sql } from "@voidhash/db"; import { Clock, Effect } from "effect"; import { describe, expect, it } from "vitest"; -import { makeSelfhostAnalyticsRuntimeLive } from "../src/backend/Analytics.ts"; +import { makeSelfhostPlatformLive } from "../src/backend/PlatformProfile.ts"; import { runSelfhostPaywallThumbnailConsumer } from "../src/backend/Thumbnails.ts"; import { getSelfhostRuntimeConfig } from "../src/config.ts"; import { @@ -56,7 +56,7 @@ describe("self-host thumbnail queue", () => { } }), ).pipe( - Effect.provide(makeSelfhostAnalyticsRuntimeLive(config)), + Effect.provide(makeSelfhostPlatformLive(config)), Effect.ensuring(cleanup), ); diff --git a/apps/www/src/routeTree.gen.ts b/apps/www/src/routeTree.gen.ts index 11539d18b..5d5bab4e2 100644 --- a/apps/www/src/routeTree.gen.ts +++ b/apps/www/src/routeTree.gen.ts @@ -57,9 +57,6 @@ import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlug import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDottrialsRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics.trials' import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotsubscribersRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics.subscribers' import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics.revenue' -import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics.query' -import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics.insights' -import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics.dashboards' import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics.churn' import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDotsentNotificationsRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/activity.sent-notifications' import { Route as StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDoteventsRouteImport } from './../../../packages/web-app/src/routes/shared/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/activity.events' @@ -404,33 +401,6 @@ const StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDot StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute, } as any, ) -const StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute = - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRouteImport.update( - { - id: '/analytics/query', - path: '/analytics/query', - getParentRoute: () => - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute, - } as any, - ) -const StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute = - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRouteImport.update( - { - id: '/analytics/insights', - path: '/analytics/insights', - getParentRoute: () => - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute, - } as any, - ) -const StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute = - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRouteImport.update( - { - id: '/analytics/dashboards', - path: '/analytics/dashboards', - getParentRoute: () => - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute, - } as any, - ) const StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute = StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRouteImport.update( { @@ -572,9 +542,6 @@ export interface FileRoutesByFullPath { '/studio/$organizationSlug/$projectSlug/activity/events': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDoteventsRoute '/studio/$organizationSlug/$projectSlug/activity/sent-notifications': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDotsentNotificationsRoute '/studio/$organizationSlug/$projectSlug/analytics/churn': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute - '/studio/$organizationSlug/$projectSlug/analytics/dashboards': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute - '/studio/$organizationSlug/$projectSlug/analytics/insights': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute - '/studio/$organizationSlug/$projectSlug/analytics/query': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute '/studio/$organizationSlug/$projectSlug/analytics/revenue': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRoute '/studio/$organizationSlug/$projectSlug/analytics/subscribers': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotsubscribersRoute '/studio/$organizationSlug/$projectSlug/analytics/trials': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDottrialsRoute @@ -629,9 +596,6 @@ export interface FileRoutesByTo { '/studio/$organizationSlug/$projectSlug/activity/events': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDoteventsRoute '/studio/$organizationSlug/$projectSlug/activity/sent-notifications': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDotsentNotificationsRoute '/studio/$organizationSlug/$projectSlug/analytics/churn': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute - '/studio/$organizationSlug/$projectSlug/analytics/dashboards': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute - '/studio/$organizationSlug/$projectSlug/analytics/insights': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute - '/studio/$organizationSlug/$projectSlug/analytics/query': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute '/studio/$organizationSlug/$projectSlug/analytics/revenue': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRoute '/studio/$organizationSlug/$projectSlug/analytics/subscribers': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotsubscribersRoute '/studio/$organizationSlug/$projectSlug/analytics/trials': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDottrialsRoute @@ -695,9 +659,6 @@ export interface FileRoutesById { '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/activity/events': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDoteventsRoute '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/activity/sent-notifications': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDotsentNotificationsRoute '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/churn': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute - '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/dashboards': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute - '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/insights': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute - '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/query': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/revenue': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRoute '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/subscribers': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotsubscribersRoute '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/trials': typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDottrialsRoute @@ -759,9 +720,6 @@ export interface FileRouteTypes { | '/studio/$organizationSlug/$projectSlug/activity/events' | '/studio/$organizationSlug/$projectSlug/activity/sent-notifications' | '/studio/$organizationSlug/$projectSlug/analytics/churn' - | '/studio/$organizationSlug/$projectSlug/analytics/dashboards' - | '/studio/$organizationSlug/$projectSlug/analytics/insights' - | '/studio/$organizationSlug/$projectSlug/analytics/query' | '/studio/$organizationSlug/$projectSlug/analytics/revenue' | '/studio/$organizationSlug/$projectSlug/analytics/subscribers' | '/studio/$organizationSlug/$projectSlug/analytics/trials' @@ -816,9 +774,6 @@ export interface FileRouteTypes { | '/studio/$organizationSlug/$projectSlug/activity/events' | '/studio/$organizationSlug/$projectSlug/activity/sent-notifications' | '/studio/$organizationSlug/$projectSlug/analytics/churn' - | '/studio/$organizationSlug/$projectSlug/analytics/dashboards' - | '/studio/$organizationSlug/$projectSlug/analytics/insights' - | '/studio/$organizationSlug/$projectSlug/analytics/query' | '/studio/$organizationSlug/$projectSlug/analytics/revenue' | '/studio/$organizationSlug/$projectSlug/analytics/subscribers' | '/studio/$organizationSlug/$projectSlug/analytics/trials' @@ -881,9 +836,6 @@ export interface FileRouteTypes { | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/activity/events' | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/activity/sent-notifications' | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/churn' - | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/dashboards' - | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/insights' - | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/query' | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/revenue' | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/subscribers' | '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/trials' @@ -1260,27 +1212,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRouteImport parentRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute } - '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/query': { - id: '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/query' - path: '/analytics/query' - fullPath: '/studio/$organizationSlug/$projectSlug/analytics/query' - preLoaderRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRouteImport - parentRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute - } - '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/insights': { - id: '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/insights' - path: '/analytics/insights' - fullPath: '/studio/$organizationSlug/$projectSlug/analytics/insights' - preLoaderRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRouteImport - parentRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute - } - '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/dashboards': { - id: '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/dashboards' - path: '/analytics/dashboards' - fullPath: '/studio/$organizationSlug/$projectSlug/analytics/dashboards' - preLoaderRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRouteImport - parentRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRoute - } '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/churn': { id: '/studio/_authenticated/_dashboard/_project/$organizationSlug/$projectSlug/analytics/churn' path: '/analytics/churn' @@ -1448,9 +1379,6 @@ interface StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRou StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDoteventsRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDoteventsRoute StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDotsentNotificationsRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDotsentNotificationsRoute StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRoute StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotsubscribersRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotsubscribersRoute StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDottrialsRoute: typeof StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDottrialsRoute @@ -1490,12 +1418,6 @@ const StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugRouteRouteCh StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugActivityDotsentNotificationsRoute, StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute: StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotchurnRoute, - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute: - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotdashboardsRoute, - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute: - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotinsightsRoute, - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute: - StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotqueryRoute, StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRoute: StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotrevenueRoute, StudioAuthenticatedDashboardProjectOrganizationSlugProjectSlugAnalyticsDotsubscribersRoute: diff --git a/docs/architecture.md b/docs/architecture.md index 9df2e5445..0277beba5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,7 +11,7 @@ repository. flowchart TD Community["voidhash Community codebase
MIT SDKs + AGPL services"] Platform["@voidhash/platform
provider-neutral contracts"] - Node["Community self-host
Node + PostgreSQL + MinIO + optional ClickHouse"] + Node["Community self-host
Node + PostgreSQL + MinIO"] Cloud["Managed Cloud
Cloudflare + PlanetScale adapters"] Private["Private composition
Enterprise + Overwatch + deployment graph"] @@ -62,8 +62,8 @@ The self-host runtime is a modular monolith. One Node process serves the API and dashboard and runs Mimic entities, queue consumers, workflows, and cron fibers. PostgreSQL provides transactional state and durable scheduling; MinIO provides S3-compatible objects; the compiler is isolated in a private-network -sidecar; Chromium renders paywall artifacts. ClickHouse is an optional -analytics profile. Community authenticates a single root account from the +sidecar; Chromium renders paywall artifacts. PostgreSQL also stores the +portable Community analytics event log. Community authenticates a single root account from the environment and needs no external identity service. See [the self-hosting guide](../selfhost/README.md) for the supported Compose diff --git a/docs/launch-announcement-draft.md b/docs/launch-announcement-draft.md index dbec5664f..6ab4442a6 100644 --- a/docs/launch-announcement-draft.md +++ b/docs/launch-announcement-draft.md @@ -17,7 +17,8 @@ deployment systems are not part of the Community repository. The self-host composition runs the same application services as Voidhash Cloud through provider-neutral platform contracts. It uses Node, PostgreSQL, MinIO, -an isolated component compiler, Chromium, SMTP, and optional ClickHouse. +an isolated component compiler, Chromium, and SMTP. PostgreSQL also stores the +Community analytics event log. Community signs in with a root account you configure in the environment and uses your own provider credentials. Cloud remains the zero-operations path; pricing is not being announced with this release. diff --git a/docs/licensing-and-self-hosting-faq.md b/docs/licensing-and-self-hosting-faq.md index 8c9c4b65f..90327d55f 100644 --- a/docs/licensing-and-self-hosting-faq.md +++ b/docs/licensing-and-self-hosting-faq.md @@ -52,12 +52,11 @@ keeps the model easy to reason about while the project is young. Multi-user self-host — most likely as a generic OIDC adapter behind the same identity port the hosted cloud uses — is a public roadmap item. -## Is analytics required? +## How does Community analytics work? -No. ClickHouse is an optional Compose profile. Without it, the platform still -boots, purchase and identity state remains durable in PostgreSQL, and analytics -queries degrade to empty results. Enable the profile for durable event capture -and dashboards. +Community stores built-in lifecycle and revenue events in PostgreSQL. The +capture endpoints remain SDK-compatible, while custom events and advanced query +features are outside the Community analytics surface. ## Are pricing or trademark terms defined here? diff --git a/docs/security/backend-threat-model.md b/docs/security/backend-threat-model.md index 1d671fc41..135a6ecee 100644 --- a/docs/security/backend-threat-model.md +++ b/docs/security/backend-threat-model.md @@ -30,8 +30,8 @@ operational objective, but not a guarantee made by the Community Edition. ## Assets and actors Protected assets include dashboard sessions, user and project API keys, payment -provider credentials, webhook signing secrets, tenant database rows, -ClickHouse events, Mimic documents and document tokens, unpublished paywall +provider credentials, webhook signing secrets, tenant database and analytics rows, +Mimic documents and document tokens, unpublished paywall artifacts, object-store credentials, and compiler/container integrity. Relevant actors are anonymous internet clients, SDK clients holding a @@ -44,7 +44,7 @@ submitting component source. 1. Internet to WWW/backend HTTP and WebSocket surfaces. 2. Authentication middleware to request-scoped `AuthSession`. -3. Tenant-scoped services to PostgreSQL and ClickHouse adapters. +3. Tenant-scoped services to PostgreSQL adapters. 4. Provider webhook ingress to provider verification and idempotent ledgers. 5. Backend to queues, workflows, object stores, SMTP, and screenshot services. 6. Backend to the component compiler container/sidecar. @@ -156,9 +156,8 @@ Current controls: project. - Foreign keys and unique constraints preserve project ownership and idempotency invariants. -- Cloud analytics queries use dedicated least-privilege users and compiler- - injected project predicates. Self-host creates separate read-write, - read-only, and analytics-query users when ClickHouse is enabled. +- Community analytics reads authorize the organization or project before + querying the portable PostgreSQL event log. - The private operations plane and staff authorization are absent from this repository and from the product backend. - Integration suites exercise forbidden access across API keys, paywalls, @@ -323,15 +322,15 @@ unchanged would compromise all stored data. Before any non-local deployment, the operator must replace every example password, set real root credentials and a real session signing secret, -configure HTTPS at the reverse proxy, restrict MinIO/Mailpit/ClickHouse host +configure HTTPS at the reverse proxy, restrict MinIO and Mailpit host ports, configure CORS and public URLs, use real SMTP credentials, back up persistent volumes, and apply host/container updates. Production mode validates configuration before migrations or the application start. It refuses missing and known example root credentials, session signing -secret, database, object-store, Mimic, and enabled ClickHouse credentials, and +secret, database, object-store, and Mimic credentials, and requires HTTPS for every public, file, and Mimic URL. Tests cover explicit mode -selection, every credential class, optional ClickHouse, and every URL +selection, every credential class, and every URL boundary. Independent review must still confirm the list remains complete as new infrastructure is added. diff --git a/docs/security/endpoint-authorization-matrix.md b/docs/security/endpoint-authorization-matrix.md index 74c84c456..c2079bb24 100644 --- a/docs/security/endpoint-authorization-matrix.md +++ b/docs/security/endpoint-authorization-matrix.md @@ -59,7 +59,7 @@ database-backed cross-tenant case. “Gap” is a publication blocker. | Group | Operations | | --- | --- | | AgentSession | `ListAgentSessions`, `GetAgentSession`, `DeleteAgentSession`, `RevertAgentEditSession`, `UploadAgentAttachment` | -| Analytics | `ListRecentAnalyticsEvents`, `QueryAnalyticsInsights`, `QueryCustomAnalyticsInsight`, `QueryCustomAnalyticsPersons`, `ListAnalyticsInsights`, `CreateAnalyticsInsight`, `UpdateAnalyticsInsight`, `DeleteAnalyticsInsight`, `ListAnalyticsCohorts`, `CreateAnalyticsCohort`, `UpdateAnalyticsCohort`, `DeleteAnalyticsCohort`, `ListAnalyticsDashboards`, `CreateAnalyticsDashboard`, `DuplicateAnalyticsDashboard`, `UpdateAnalyticsDashboard`, `DeleteAnalyticsDashboard`, `PutAnalyticsDashboardItem`, `ReorderAnalyticsDashboardItems`, `RemoveAnalyticsDashboardItem` | +| Analytics | `ListRecentAnalyticsEvents`, `QueryAnalyticsInsights` | | ApiKey | `CreateSecretKey`, `ListApiKeys`, `GetApiKeyById`, `RotateSecretKey`, `DeleteApiKey`, `CreateUserApiKey`, `ListUserApiKeys`, `RevokeUserApiKey` | | Person | `CreatePerson`, `ListPersons`, `GetPersonById`, `GetPersonByDistinctId` | | Experiment | `ListExperiments`, `GetExperiment`, `CreateExperiment`, `SaveExperimentSetup`, `StartExperiment`, `PauseExperiment`, `ConcludeExperiment`, `ArchiveExperiment`, `RestoreExperiment`, `GetExperimentResults` | @@ -81,13 +81,12 @@ database-backed cross-tenant case. “Gap” is a publication blocker. | Product | `ListProducts`, `GetProduct`, `CreateProduct`, `UpdateProduct`, `DeleteProduct` | | Project | `CreateProject`, `ListProjects`, `UpdateProject`, `DeleteProject`, `SetProjectAvatar`, `RemoveProjectAvatar` | | User | `CurrentUser`, `SetUserAvatar`, `RemoveUserAvatar` | -| VoidQl | `RunVoidQlQuery`, `ValidateVoidQlQuery`, `GetVoidQlSchema`, `SaveVoidQlInsight`, `ListVoidQlInsights`, `RunSavedVoidQlInsight`, `DeleteVoidQlInsight` | | Webhook | `ListWebhookEndpoints`, `GetWebhookEndpoint`, `CreateWebhookEndpoint`, `UpdateWebhookEndpoint`, `DeleteWebhookEndpoint`, `RotateWebhookSecret`, `TestWebhookEndpoint`, `ListWebhookDeliveries`, `GetWebhookDelivery`, `RetryWebhookDelivery` | | RPC groups | Authorization boundary | Evidence | Status | | --- | --- | --- | --- | -| Analytics, VoidQl | Project permission before query compilation/execution; compiled SQL carries a bound tenant predicate. | Analytics integration suite and VoidQL compiler/substrate tests | Integrated | +| Analytics | Project permission before event reads or built-in revenue queries; PostgreSQL reads remain project-scoped. | Community PostgreSQL analytics integration suite | Integrated | | ApiKey, Person, Organization, PaymentProviderConfiguration, PaymentProviderProduct, PaywallDeploy, PaywallLocation, Paywall, Perk, ProductPerk, Product, Project, Webhook, FeatureFlag | Project/organization permission followed by stored ownership checks for nested IDs. | Corresponding database-backed core service integration suites | Integrated | | PaywallComponent | Delegates to the project-authorized deploy service. | Paywall deploy integration suite | Integrated | | AgentSession, PaywallAsset, PaywallWorkspace | Project membership/permission and stored parent ownership are checked before every read or mutation. Client-minted session ID collisions are bound to the persisted user and project scope. | `AgentSessionIndexService.test.ts`, `agent-session-rpcs.test.ts`, `PaywallAssetAuthorization.integration.test.ts`, `PaywallWorkspaceAuthorization.integration.test.ts`, plus service and RPC unit tests | Integrated | diff --git a/package.json b/package.json index f309b457e..0fbdd2661 100644 --- a/package.json +++ b/package.json @@ -54,8 +54,8 @@ "check:platform-seam": "node ./scripts/check-platform-seam.mjs", "check:selfhost-runtime": "node ./scripts/check-selfhost-runtime-boundary.mjs", "check:test-tiers": "node ./scripts/check-test-tiers.mjs", - "stack:up": "SELFHOST_MODE=local-evaluation docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env --profile analytics --project-directory selfhost up -d --build", - "stack:down": "SELFHOST_MODE=local-evaluation docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env --profile analytics --project-directory selfhost down", + "stack:up": "SELFHOST_MODE=local-evaluation docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env --project-directory selfhost up -d --build", + "stack:down": "SELFHOST_MODE=local-evaluation docker compose -f selfhost/docker-compose.yml -f selfhost/docker-compose.dev.yml --env-file .env --project-directory selfhost down", "verify": "pnpm verify:quick && pnpm test:integration && pnpm test:e2e && pnpm test:e2e:release", "verify:quick": "pnpm check:publication && pnpm check:platform-seam && pnpm check:selfhost-runtime && pnpm check:test-tiers && pnpm lint && pnpm typecheck && pnpm test", "test": "turbo test", diff --git a/packages/backend/package.json b/packages/backend/package.json index ce09c597d..edb95afe9 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -31,7 +31,6 @@ "@voidhash/ai-shared": "workspace:*", "@voidhash/api-contracts": "workspace:*", "@voidhash/app-store-server-sdk": "workspace:*", - "@voidhash/clickhouse-db": "workspace:*", "@voidhash/core": "workspace:*", "@voidhash/db": "workspace:*", "@voidhash/google-play-server-sdk": "workspace:*", diff --git a/packages/backend/src/BackendApp.ts b/packages/backend/src/BackendApp.ts index 35acb4d85..04b0b96ee 100644 --- a/packages/backend/src/BackendApp.ts +++ b/packages/backend/src/BackendApp.ts @@ -1,12 +1,12 @@ import { AppStoreServerSdk } from "@voidhash/app-store-server-sdk"; import { VoidhashV1Api } from "@voidhash/api-contracts"; import { Db } from "@voidhash/db"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; import { PaymentConfigSecretCrypto } from "@voidhash/core/utils/crypto/PaymentConfigSecretCrypto"; import { PaywallAssetConfig } from "@voidhash/core/services/paywallLocations/PaywallAssetConfig"; import { IdentityProvider } from "@voidhash/core/services/auth/IdentityProvider"; import { AnalyticsService, + AnalyticsEventStore, ApiKeyService, AuditLogPort, AppStorePaymentProvider, @@ -55,7 +55,6 @@ import { PaywallWorkspaceService, ComponentCompiler, ComponentManifestCacheService, - CustomAnalyticsService, PerkGrantService, PerkService, PersonIdentityService, @@ -79,12 +78,10 @@ import { StripePaymentProviderConfigLive, StripePaymentProviderServiceLive, UserService, - VoidQlService, WebhookManagerService, IdentityLinkBackfillService, OrgDirectoryPort, } from "@voidhash/core/services"; -import { AnalyticsWriterService } from "@voidhash/core/services/analyticsIngest/AnalyticsWriterService"; import { createInitialPaywallDocumentInput, PaywallDesignerDocument } from "@voidhash/mimic-schema"; import { AuthMiddleware } from "@voidhash/rpc"; import { constant } from "@voidhash/lib/lang"; @@ -158,7 +155,6 @@ import { ProductPerkRpcsLive } from "./rpcs/product-perk-rpcs.ts"; import { ProductRpcsLive } from "./rpcs/product-rpcs.ts"; import { ProjectRpcsLive } from "./rpcs/project-rpcs.ts"; import { UserRpcsLive } from "./rpcs/user-rpcs.ts"; -import { VoidQlRpcsLive } from "./rpcs/voidql-rpcs.ts"; import { WebhookRpcsLive } from "./rpcs/webhook-rpcs.ts"; /** @@ -167,10 +163,8 @@ import { WebhookRpcsLive } from "./rpcs/webhook-rpcs.ts"; * `Layer.provide` / `HttpRouter.provideRequest` in the route graph is actually * satisfied — without it, a missing service (e.g. the raw WorkOS webhook * handler's `Db`) only surfaces as a runtime "Service not found" instead of a - * compile error. ClickHouse is deliberately not mandatory: analytics services - * use it when the runtime's layer provides it and degrade to empty results when - * absent. The caller's bound `InfraLayer` may be structurally wider than this - * contract, so the cloud composition can still carry its analytics clients. + * compile error. Analytics storage is supplied through service layers, so the + * infrastructure contract remains database-agnostic. */ export type InfraServices = | Db @@ -217,18 +211,11 @@ export interface BackendRuntimeLayers< */ readonly mcpOAuth?: Layer.Layer; readonly infrastructure: Layer.Layer; + /** Overrides the community PostgreSQL analytics reader for hosted runtimes. */ + readonly analyticsService?: Layer.Layer; readonly features: BackendFeatureComposition; readonly routes?: Layer.Layer; readonly webhookManager?: Layer.Layer; - /** - * The hardened, single-shared `analytics_query` ClickHouse client that backs the - * VoidQL read path (`readonly = 1` CONST profile, SELECT-only, no row policy — - * isolation is the compiler-injected bound predicate). When omitted, - * {@link VoidQlService} resolves the ambient (RLS readonly) client from - * `infrastructure`, which fail-closes to empty rows because VoidQL sets no - * `SQL_organization_id`. - */ - readonly analyticsQueryClient?: Layer.Layer; /** * Queue-backed push-delivery dispatcher. Defaults to {@link PushDeliveryDispatch.noop} * (dev/smoke — rows are created but never delivered); the production worker @@ -740,9 +727,9 @@ const buildBackendServiceGraph = < | "features" | "infrastructure" | "webhookManager" - | "analyticsQueryClient" | "pushDeliveryDispatch" | "mcpOAuth" + | "analyticsService" >, ) => { const RpcHandlersLayer = Layer.mergeAll( @@ -769,7 +756,6 @@ const buildBackendServiceGraph = < ProductRpcsLive, ProjectRpcsLive, UserRpcsLive, - VoidQlRpcsLive, WebhookRpcsLive, ); @@ -796,19 +782,6 @@ const buildBackendServiceGraph = < Layer.provide(FeatureFlagService.layer), ); - // VoidQL runs under the hardened single-shared `analytics_query` user when the - // caller wires that client (Layer.provide satisfies its ClickhouseWebClient - // before the ambient RLS readonly client is merged in); without it, it resolves - // the ambient readonly client and fail-closes to empty rows. - const voidQlServiceLayer = () => { - const analyticsQueryClient = layers.analyticsQueryClient; - if (analyticsQueryClient) { - return VoidQlService.layer.pipe(Layer.provide(analyticsQueryClient)); - } - return VoidQlService.layer; - }; - const VoidQlServiceLive = voidQlServiceLayer(); - const PaywallWorkspaceServiceLive = PaywallWorkspaceService.layer.pipe( Layer.provide(PaywallService.layer), Layer.provide(ComponentManifestCacheService.layer), @@ -819,11 +792,11 @@ const buildBackendServiceGraph = < ); const AgentSessionIndexServiceLive = AgentSessionIndexService.layer; + const AnalyticsEventStoreLive = AnalyticsEventStore.layer; const BaseDomainServicesLayer = Layer.mergeAll( AgentSessionIndexServiceLive, AgentAttachmentService.layer.pipe(Layer.provide(AgentSessionIndexServiceLive)), - AnalyticsService.layer, - CustomAnalyticsService.layer, + layers.analyticsService ?? AnalyticsService.layer.pipe(Layer.provide(AnalyticsEventStoreLive)), ApiKeyService.layer, BackendFeedbackServiceLive, BackendAppStorePaymentProviderServiceLive, @@ -869,25 +842,12 @@ const buildBackendServiceGraph = < PurchaseService.layer, SchemaService.layer, UserService.layer, - VoidQlServiceLive, layers.webhookManager ?? WebhookManagerService.layer, ).pipe(Layer.provide(BackendPushProvidersLive), Layer.provide(SupportServicesLayer)); - // The synchronous SDK person-attribute write projects into ClickHouse via the - // real `analyticsWriterLayer`. This is scoped to `SdkService` ONLY (provided - // innermost so it discharges the `IdentityProjectionPublisher` requirement - // first) — everywhere else, including `PersonIdentityService`'s own publisher - // and the async ingest processor, keeps the no-op binding so person rows are - // never double-written. - const SdkIdentityProjectionPublisherLayer = IdentityProjectionPublisher.analyticsWriterLayer.pipe( - Layer.provide(AnalyticsWriterService.layer), - Layer.provide(layers.infrastructure), - ); - const DomainServicesLayer = Layer.mergeAll( BaseDomainServicesLayer, SdkService.layer.pipe( - Layer.provide(SdkIdentityProjectionPublisherLayer), Layer.provide(BaseDomainServicesLayer), Layer.provide(SupportServicesLayer), ), @@ -934,9 +894,9 @@ export const buildBackendAgentServices = < | "features" | "infrastructure" | "webhookManager" - | "analyticsQueryClient" | "pushDeliveryDispatch" | "mcpOAuth" + | "analyticsService" >, ) => { const graph = buildBackendServiceGraph(layers); @@ -1016,7 +976,7 @@ export const buildBackendRpcServices = < | "features" | "infrastructure" | "webhookManager" - | "analyticsQueryClient" + | "analyticsService" | "rpcExtension" | "mcpOAuth" >, diff --git a/packages/backend/src/rpc-smoke.integration.test.ts b/packages/backend/src/rpc-smoke.integration.test.ts index 831a95a53..e654d739c 100644 --- a/packages/backend/src/rpc-smoke.integration.test.ts +++ b/packages/backend/src/rpc-smoke.integration.test.ts @@ -12,7 +12,7 @@ * - **Raw route cases** feed synthetic requests to the real * `buildBackendFetch` route graph via a synthetic `HttpServerRequest`. * - * Infra is real where the deployed stack is real: `Db`/`Clickhouse` + * Infra is real where the deployed stack is real: PostgreSQL * are built from the gated `testConnections`. Only the genuine external/platform * seams are doubled — `OrgDirectoryPort` is faked so organization RPCs never * touch a real directory; payment providers use local stubs; the webhook manager @@ -28,7 +28,6 @@ import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse"; import { RpcClient, RpcTest } from "effect/unstable/rpc"; import { describe, expect, inject, test } from "vitest"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; import { PaywallArtifactStore } from "@voidhash/core/services"; import { StandaloneIdentityProviderLive } from "@voidhash/core/services/auth/StandaloneIdentityProvider"; import { Db } from "@voidhash/db"; @@ -50,7 +49,6 @@ import { } from "./BackendApp.ts"; import { BackendRpcGroups as RpcGroups } from "./BackendRpcGroups.ts"; import { - TestClickhouseLive, TestProjectSchemaCacheLive, TestWebhookManagerServiceLive, TestWorkflowRunnerLive, @@ -120,14 +118,13 @@ const SmokePaywallArtifactStoreLive = Layer.sync(PaywallArtifactStore, () => { }); /** - * In-process infrastructure for the RPC handler graph: real `Db`/`Clickhouse` + * In-process infrastructure for the RPC handler graph: real PostgreSQL * from the deployed stack, an in-memory schema cache, a faked * `OrgDirectoryPort`, and local payment/paywall/identity stubs. */ const makeRpcInfra = (tc: BackendTestConnections) => Layer.mergeAll( Db.layer(tc.db), - ClickhouseWebClient.layer(tc.clickhouse).pipe(Layer.orDie), StandaloneIdentityProviderLive(SMOKE_AUTH_SECRET), TestProjectSchemaCacheLive, TestOrgDirectoryLive, @@ -143,13 +140,11 @@ const makeRpcInfra = (tc: BackendTestConnections) => /** * In-process infrastructure for the raw route graph: real `Db` plus the same - * stubs as the RPC graph (these routes need no ClickHouse, so a no-op stands - * in). + * stubs as the RPC graph. */ const makeRouteInfra = (tc: BackendTestConnections) => Layer.mergeAll( Db.layer(tc.db), - TestClickhouseLive, TestProjectSchemaCacheLive, TestOrgDirectoryLive, BackendMimicHostStubLive, diff --git a/packages/backend/src/rpcs/analytics-rpcs.ts b/packages/backend/src/rpcs/analytics-rpcs.ts index 122d21787..6f8c0284b 100644 --- a/packages/backend/src/rpcs/analytics-rpcs.ts +++ b/packages/backend/src/rpcs/analytics-rpcs.ts @@ -1,4 +1,4 @@ -import { AnalyticsService, CustomAnalyticsService } from "@voidhash/core/services"; +import { AnalyticsService } from "@voidhash/core/services"; import { AnalyticsRpcsDef, RpcActionForbiddenError, @@ -11,49 +11,27 @@ import { } from "@voidhash/rpc"; import { Effect } from "effect"; -/** Copies a readonly breakdown list into the mutable array the service expects. */ const toMutableBreakdowns = (breakdowns: readonly T[] | undefined): T[] | undefined => { if (!breakdowns) return undefined; return [...breakdowns]; }; +/** Community analytics handlers: recent events and built-in revenue insights. */ export const AnalyticsRpcsLive = AnalyticsRpcsDef.toLayer( Effect.gen(function* AnalyticsRpcsLive() { const analyticsService = yield* AnalyticsService; - const customAnalyticsService = yield* CustomAnalyticsService; - - const commonErrors = { - ActionForbiddenError: (error: { readonly message: string }) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - AnalyticsServiceError: (error: { readonly cause: string; readonly message: string }) => - Effect.fail( - new RpcAnalyticsServiceError({ - cause: error.cause, - message: error.message, - }), - ), - }; - return { ListRecentAnalyticsEvents: ({ projectId, limit }) => - analyticsService - .listRecentEvents({ - limit, - projectId, - }) - .pipe( - Effect.catchTags({ - ActionForbiddenError: (error) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - AnalyticsServiceError: (error) => - Effect.fail( - new RpcAnalyticsServiceError({ - cause: error.cause, - message: error.message, - }), - ), - }), - ), + analyticsService.listRecentEvents({ limit, projectId }).pipe( + Effect.catchTags({ + ActionForbiddenError: (error) => + Effect.fail(new RpcActionForbiddenError({ message: error.message })), + AnalyticsServiceError: (error) => + Effect.fail( + new RpcAnalyticsServiceError({ cause: error.cause, message: error.message }), + ), + }), + ), QueryAnalyticsInsights: ({ queries }) => analyticsService .queryAnalyticsInsights({ @@ -68,10 +46,7 @@ export const AnalyticsRpcsLive = AnalyticsRpcsDef.toLayer( Effect.fail(new RpcActionForbiddenError({ message: error.message })), AnalyticsServiceError: (error) => Effect.fail( - new RpcAnalyticsServiceError({ - cause: error.cause, - message: error.message, - }), + new RpcAnalyticsServiceError({ cause: error.cause, message: error.message }), ), InvalidAnalyticsQueryError: (error) => Effect.fail(new RpcInvalidAnalyticsQueryError({ message: error.message })), @@ -100,58 +75,6 @@ export const AnalyticsRpcsLive = AnalyticsRpcsDef.toLayer( ), }), ), - QueryCustomAnalyticsInsight: (input) => - customAnalyticsService.queryInsight(input).pipe( - Effect.catchTags({ - ...commonErrors, - InvalidAnalyticsQueryError: (error) => - Effect.fail(new RpcInvalidAnalyticsQueryError({ message: error.message })), - InvalidTimeRangeError: (error) => - Effect.fail(new RpcInvalidTimeRangeError({ message: error.message })), - }), - ), - QueryCustomAnalyticsPersons: (input) => - customAnalyticsService.queryPersons(input).pipe( - Effect.catchTags({ - ...commonErrors, - InvalidAnalyticsQueryError: (error) => - Effect.fail(new RpcInvalidAnalyticsQueryError({ message: error.message })), - InvalidTimeRangeError: (error) => - Effect.fail(new RpcInvalidTimeRangeError({ message: error.message })), - }), - ), - ListAnalyticsInsights: (input) => - customAnalyticsService.listInsights(input).pipe(Effect.catchTags(commonErrors)), - CreateAnalyticsInsight: (input) => - customAnalyticsService.createInsight(input).pipe(Effect.catchTags(commonErrors)), - UpdateAnalyticsInsight: (input) => - customAnalyticsService.updateInsight(input).pipe(Effect.catchTags(commonErrors)), - DeleteAnalyticsInsight: (input) => - customAnalyticsService.deleteInsight(input).pipe(Effect.catchTags(commonErrors)), - ListAnalyticsCohorts: (input) => - customAnalyticsService.listCohorts(input).pipe(Effect.catchTags(commonErrors)), - CreateAnalyticsCohort: (input) => - customAnalyticsService.createCohort(input).pipe(Effect.catchTags(commonErrors)), - UpdateAnalyticsCohort: (input) => - customAnalyticsService.updateCohort(input).pipe(Effect.catchTags(commonErrors)), - DeleteAnalyticsCohort: (input) => - customAnalyticsService.deleteCohort(input).pipe(Effect.catchTags(commonErrors)), - ListAnalyticsDashboards: (input) => - customAnalyticsService.listDashboards(input).pipe(Effect.catchTags(commonErrors)), - CreateAnalyticsDashboard: (input) => - customAnalyticsService.createDashboard(input).pipe(Effect.catchTags(commonErrors)), - DuplicateAnalyticsDashboard: (input) => - customAnalyticsService.duplicateDashboard(input).pipe(Effect.catchTags(commonErrors)), - UpdateAnalyticsDashboard: (input) => - customAnalyticsService.updateDashboard(input).pipe(Effect.catchTags(commonErrors)), - DeleteAnalyticsDashboard: (input) => - customAnalyticsService.deleteDashboard(input).pipe(Effect.catchTags(commonErrors)), - PutAnalyticsDashboardItem: (input) => - customAnalyticsService.putDashboardItem(input).pipe(Effect.catchTags(commonErrors)), - ReorderAnalyticsDashboardItems: (input) => - customAnalyticsService.reorderDashboardItems(input).pipe(Effect.catchTags(commonErrors)), - RemoveAnalyticsDashboardItem: (input) => - customAnalyticsService.removeDashboardItem(input).pipe(Effect.catchTags(commonErrors)), }; }), ); diff --git a/packages/backend/src/rpcs/voidql-rpcs.ts b/packages/backend/src/rpcs/voidql-rpcs.ts deleted file mode 100644 index 9e3b7b422..000000000 --- a/packages/backend/src/rpcs/voidql-rpcs.ts +++ /dev/null @@ -1,205 +0,0 @@ -/** - * Backend handlers for the VoidQL RPC group. Each handler calls - * {@link VoidQlService} and translates the core `VoidQl*` domain errors into - * their `Rpc/`-prefixed counterparts. The internal `VoidQlIsolationError` (a - * compiler defect) is mapped to the opaque {@link RpcVoidQlExecutionError} so it - * never leaks a reason to the client. - * - * NOTE: VoidQL must execute under the locked-down `analytics_query` ClickHouse - * user; {@link VoidQlService} reads the ambient `ClickhouseWebClient`, so the - * worker provides this layer the `analyticsQuery` client (see BackendWorker). - */ -import { VoidQlService } from "@voidhash/core/services"; -import { - AuthSession, - RpcActionForbiddenError, - RpcVoidQlComplexityError, - RpcVoidQlExecutionError, - RpcVoidQlPiiError, - RpcVoidQlSchemaError, - RpcVoidQlSyntaxError, - RpcVoidQlUnknownFieldError, - RpcVoidQlUnsupportedError, - VoidQlRpcsDef, -} from "@voidhash/rpc"; -import { Effect } from "effect"; - -export const VoidQlRpcsLive = VoidQlRpcsDef.toLayer( - Effect.gen(function* VoidQlRpcsLive() { - const voidql = yield* VoidQlService; - - return { - RunVoidQlQuery: ({ organizationId, text }) => - Effect.gen(function* () { - const session = yield* AuthSession; - return yield* voidql.runQuery({ - organizationId, - text, - principal: { kind: "user", id: session?.user?.id ?? "api-key" }, - }); - }).pipe( - Effect.catchTags({ - ActionForbiddenError: (error) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - VoidQlSyntaxError: (error) => - Effect.fail(new RpcVoidQlSyntaxError({ message: error.message, hint: error.hint })), - VoidQlUnsupportedError: (error) => - Effect.fail( - new RpcVoidQlUnsupportedError({ message: error.message, hint: error.hint }), - ), - VoidQlSchemaError: (error) => - Effect.fail(new RpcVoidQlSchemaError({ message: error.message })), - VoidQlUnknownFieldError: (error) => - Effect.fail( - new RpcVoidQlUnknownFieldError({ - field: error.field, - message: error.message, - suggestion: error.suggestion, - }), - ), - VoidQlPiiError: (error) => - Effect.fail(new RpcVoidQlPiiError({ message: error.message })), - VoidQlComplexityError: (error) => - Effect.fail(new RpcVoidQlComplexityError({ message: error.message })), - VoidQlIsolationError: () => - Effect.fail( - new RpcVoidQlExecutionError({ - cause: "internal", - message: "The query could not be executed.", - }), - ), - VoidQlExecutionError: (error) => - Effect.fail( - new RpcVoidQlExecutionError({ cause: error.cause, message: error.message }), - ), - }), - ), - - ValidateVoidQlQuery: ({ organizationId, text }) => - Effect.gen(function* () { - const session = yield* AuthSession; - return yield* voidql.validateQuery({ - organizationId, - text, - principal: { kind: "user", id: session?.user?.id ?? "api-key" }, - }); - }).pipe( - Effect.catchTags({ - ActionForbiddenError: (error) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - VoidQlExecutionError: (error) => - Effect.fail( - new RpcVoidQlExecutionError({ cause: error.cause, message: error.message }), - ), - }), - ), - - GetVoidQlSchema: () => voidql.getSchema(), - - SaveVoidQlInsight: ({ organizationId, name, text }) => - voidql.saveInsight({ organizationId, name, text }).pipe( - Effect.catchTags({ - ActionForbiddenError: (error) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - VoidQlSyntaxError: (error) => - Effect.fail(new RpcVoidQlSyntaxError({ message: error.message, hint: error.hint })), - VoidQlUnsupportedError: (error) => - Effect.fail( - new RpcVoidQlUnsupportedError({ message: error.message, hint: error.hint }), - ), - VoidQlSchemaError: (error) => - Effect.fail(new RpcVoidQlSchemaError({ message: error.message })), - VoidQlUnknownFieldError: (error) => - Effect.fail( - new RpcVoidQlUnknownFieldError({ - field: error.field, - message: error.message, - suggestion: error.suggestion, - }), - ), - VoidQlPiiError: (error) => - Effect.fail(new RpcVoidQlPiiError({ message: error.message })), - VoidQlComplexityError: (error) => - Effect.fail(new RpcVoidQlComplexityError({ message: error.message })), - VoidQlIsolationError: () => - Effect.fail( - new RpcVoidQlExecutionError({ - cause: "internal", - message: "The insight could not be saved.", - }), - ), - VoidQlExecutionError: (error) => - Effect.fail( - new RpcVoidQlExecutionError({ cause: error.cause, message: error.message }), - ), - }), - ), - ListVoidQlInsights: (input) => - voidql.listInsights(input).pipe( - Effect.catchTags({ - ActionForbiddenError: (error) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - VoidQlExecutionError: (error) => - Effect.fail( - new RpcVoidQlExecutionError({ cause: error.cause, message: error.message }), - ), - }), - ), - RunSavedVoidQlInsight: ({ id }) => - Effect.gen(function* () { - const session = yield* AuthSession; - return yield* voidql.runSavedInsight({ - id, - principal: { kind: "user", id: session?.user?.id ?? "api-key" }, - }); - }).pipe( - Effect.catchTags({ - ActionForbiddenError: (error) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - VoidQlSyntaxError: (error) => - Effect.fail(new RpcVoidQlSyntaxError({ message: error.message, hint: error.hint })), - VoidQlUnsupportedError: (error) => - Effect.fail( - new RpcVoidQlUnsupportedError({ message: error.message, hint: error.hint }), - ), - VoidQlSchemaError: (error) => - Effect.fail(new RpcVoidQlSchemaError({ message: error.message })), - VoidQlUnknownFieldError: (error) => - Effect.fail( - new RpcVoidQlUnknownFieldError({ - field: error.field, - message: error.message, - suggestion: error.suggestion, - }), - ), - VoidQlPiiError: (error) => - Effect.fail(new RpcVoidQlPiiError({ message: error.message })), - VoidQlComplexityError: (error) => - Effect.fail(new RpcVoidQlComplexityError({ message: error.message })), - VoidQlIsolationError: () => - Effect.fail( - new RpcVoidQlExecutionError({ - cause: "internal", - message: "The query could not be executed.", - }), - ), - VoidQlExecutionError: (error) => - Effect.fail( - new RpcVoidQlExecutionError({ cause: error.cause, message: error.message }), - ), - }), - ), - DeleteVoidQlInsight: (input) => - voidql.deleteInsight(input).pipe( - Effect.catchTags({ - ActionForbiddenError: (error) => - Effect.fail(new RpcActionForbiddenError({ message: error.message })), - VoidQlExecutionError: (error) => - Effect.fail( - new RpcVoidQlExecutionError({ cause: error.cause, message: error.message }), - ), - }), - ), - }; - }), -); diff --git a/packages/backend/src/testing/BackendTestConnections.ts b/packages/backend/src/testing/BackendTestConnections.ts index bf8743e46..a28e13fc5 100644 --- a/packages/backend/src/testing/BackendTestConnections.ts +++ b/packages/backend/src/testing/BackendTestConnections.ts @@ -7,12 +7,6 @@ export interface BackendTestConnections { readonly password: string; readonly databaseName: string; }; - readonly clickhouse: { - readonly url: string; - readonly username: string; - readonly password: string; - readonly database: string; - }; readonly workos: { readonly apiKey: string; readonly clientId: string; diff --git a/packages/backend/src/testing/TestLayers.ts b/packages/backend/src/testing/TestLayers.ts index d302b550b..723026ef9 100644 --- a/packages/backend/src/testing/TestLayers.ts +++ b/packages/backend/src/testing/TestLayers.ts @@ -3,7 +3,6 @@ import { WebhookEndpointNotFoundError, WebhookValidationError, } from "@voidhash/core/domain/webhook/Webhook"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; import * as TestWorkflowRunner from "@voidhash/platform/TestWorkflowRunner"; import { WorkflowRunner } from "@voidhash/platform/WorkflowRunner"; import { @@ -38,30 +37,6 @@ import { } from "../BackendApp.ts"; import { smokeIdsFromEmail } from "./smoke-ids.ts"; -// The webhook path that consumes this stub never calls ClickHouse, so a Proxy -// that dies on any access is a safe placeholder that satisfies the requirement -// without standing up a real client. `ClickhouseWebClient` extends the whole -// `SqlClient` surface, so no hand-written object can inhabit it — the single -// unsafe conversion lives here, in one narrow helper. -const unusableStub = (message: string): any => - new Proxy( - {}, - { - get() { - return Effect.runSync(Effect.die(new Error(message))); - }, - }, - ); - -const clickhouseStub: ClickhouseWebClient.ClickhouseWebClient = unusableStub( - "ClickhouseWebClient must not be used in this test", -); - -export const TestClickhouseLive = Layer.succeed( - ClickhouseWebClient.ClickhouseWebClient, - clickhouseStub, -); - /** Recording workflow runner used by backend smoke tests. */ export const TestWorkflowRunnerLive = Layer.succeed(WorkflowRunner, TestWorkflowRunner.make()); @@ -447,7 +422,6 @@ export const TestWebhookManagerServiceLive = Layer.effect( ); export const TestBackendStubInfrastructureLive = Layer.mergeAll( - TestClickhouseLive, TestProjectSchemaCacheLive, TestOrgDirectoryLive, BackendMimicHostStubLive, diff --git a/packages/backend/src/testing/rpc-smoke-cases.ts b/packages/backend/src/testing/rpc-smoke-cases.ts index 1db51b88a..0e1cbb5c8 100644 --- a/packages/backend/src/testing/rpc-smoke-cases.ts +++ b/packages/backend/src/testing/rpc-smoke-cases.ts @@ -832,31 +832,6 @@ const knownMissingRpcSmokeTags = new Set([ "RecordComponentManifest", "SetUserAvatar", "RemoveUserAvatar", - "RunVoidQlQuery", - "ValidateVoidQlQuery", - "GetVoidQlSchema", - "SaveVoidQlInsight", - "ListVoidQlInsights", - "RunSavedVoidQlInsight", - "DeleteVoidQlInsight", - "QueryCustomAnalyticsInsight", - "QueryCustomAnalyticsPersons", - "ListAnalyticsInsights", - "CreateAnalyticsInsight", - "UpdateAnalyticsInsight", - "DeleteAnalyticsInsight", - "ListAnalyticsCohorts", - "CreateAnalyticsCohort", - "UpdateAnalyticsCohort", - "DeleteAnalyticsCohort", - "ListAnalyticsDashboards", - "CreateAnalyticsDashboard", - "UpdateAnalyticsDashboard", - "DeleteAnalyticsDashboard", - "DuplicateAnalyticsDashboard", - "PutAnalyticsDashboardItem", - "ReorderAnalyticsDashboardItems", - "RemoveAnalyticsDashboardItem", ]); /** diff --git a/packages/clickhouse-db/LICENSE.md b/packages/clickhouse-db/LICENSE.md deleted file mode 100644 index be3f7b28e..000000000 --- a/packages/clickhouse-db/LICENSE.md +++ /dev/null @@ -1,661 +0,0 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. - - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. - - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. - - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. diff --git a/packages/clickhouse-db/package.json b/packages/clickhouse-db/package.json deleted file mode 100644 index aa0cca2a2..000000000 --- a/packages/clickhouse-db/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "@voidhash/clickhouse-db", - "version": "0.0.1-alpha.1", - "private": true, - "license": "AGPL-3.0-only", - "type": "module", - "main": "./index.ts", - "exports": { - ".": "./src/index.ts", - "./clickhouse-client-web": "./src/clickhouse-client-web/index.ts", - "./analytics/schema": "./src/analytics/schema.ts", - "./analytics/migration": "./src/analytics/migration.ts", - "./analytics/resolved-events-sql": "./src/analytics/resolved-events-sql.ts" - }, - "scripts": { - "typecheck": "tsc --noEmit", - "typecheck-go": "tsgo --noEmit", - "test": "vp test run -c vitest.mts" - }, - "dependencies": { - "@clickhouse/client-web": "^1.12.0", - "@effect/platform-bun": "catalog:", - "@voidhash/lib": "workspace:*" - }, - "devDependencies": { - "@voidhash/tsconfig": "workspace:*", - "typescript": "catalog:" - }, - "peerDependencies": { - "effect": "catalog:" - } -} diff --git a/packages/clickhouse-db/src/analytics/migration.ts b/packages/clickhouse-db/src/analytics/migration.ts deleted file mode 100644 index 82d972dea..000000000 --- a/packages/clickhouse-db/src/analytics/migration.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { constant } from "@voidhash/lib/lang"; - -import type { MigrationSet } from "../live.ts"; - -import migration0001 from "../migrations/0001_create_analytics_events.ts"; -import migration0002 from "../migrations/0002_create_analytics_identity_v2.ts"; -import migration0003 from "../migrations/0003_create_analytics_identity_pending_overrides_v2.ts"; -import migration0004 from "../migrations/0004_person_identity_cutover.ts"; -import migration0005 from "../migrations/0005_align_analytics_identity_tables.ts"; -import migration0006 from "../migrations/0006_repair_pending_overrides_person_id.ts"; -import migration0007 from "../migrations/0007_add_organization_id_to_identity_tables.ts"; -import migration0008 from "../migrations/0008_repair_identity_organization_id.ts"; - -export const analyticsEventsMigrations: MigrationSet = { - migrations: [ - [1, "0001_create_analytics_events", migration0001], - [2, "0002_create_analytics_identity_v2", migration0002], - [3, "0003_create_analytics_identity_pending_overrides_v2", migration0003], - [4, "0004_person_identity_cutover", migration0004], - [5, "0005_align_analytics_identity_tables", migration0005], - [6, "0006_repair_pending_overrides_person_id", migration0006], - [7, "0007_add_organization_id_to_identity_tables", migration0007], - [8, "0008_repair_identity_organization_id", migration0008], - ], - tableName: "analytics_migrations", -}; - -export const analyticsEventsMigrationRef = constant({ - name: "analyticsEvents", - tableName: analyticsEventsMigrations.tableName, - version: analyticsEventsMigrations.migrations.reduce((max, [id]) => Math.max(max, id), -1), -}); - -export type MigrationSetRef = typeof analyticsEventsMigrationRef; diff --git a/packages/clickhouse-db/src/analytics/resolved-events-sql.test.ts b/packages/clickhouse-db/src/analytics/resolved-events-sql.test.ts deleted file mode 100644 index 5f0ee9231..000000000 --- a/packages/clickhouse-db/src/analytics/resolved-events-sql.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { buildResolvedEventsSql } from "./resolved-events-sql.ts"; - -describe("buildResolvedEventsSql", () => { - it("falls back to raw event columns when there is no pending override", () => { - const sql = buildResolvedEventsSql(); - - expect(sql.effectivePersonIdExpression).toBe( - "coalesce(pending_overrides.person_id, events.person_id)", - ); - expect(sql.effectiveDistinctIdExpression).toBe( - "coalesce(pending_overrides.target_distinct_id, events.distinct_id)", - ); - expect(sql.fromClause).toContain("FROM events_v2 AS events"); - }); - - it("prefers pending override person ids for unsquashed events", () => { - const sql = buildResolvedEventsSql({ - eventsAlias: "raw_events", - overridesAlias: "overrides", - }); - - expect(sql.effectivePersonIdExpression).toBe( - "coalesce(overrides.person_id, raw_events.person_id)", - ); - expect(sql.leftJoinClause).toContain( - "AND overrides.source_distinct_id = raw_events.distinct_id", - ); - }); - - it("exposes a canonical distinct id without rewriting the raw event id", () => { - const sql = buildResolvedEventsSql(); - - expect(sql.effectiveDistinctIdExpression).toBe( - "coalesce(pending_overrides.target_distinct_id, events.distinct_id)", - ); - expect(sql.leftJoinClause).toContain("target_distinct_id"); - }); - - it("resolves only the latest version per raw distinct id", () => { - const sql = buildResolvedEventsSql(); - - expect(sql.leftJoinClause).toContain("LIMIT 1 BY project_id, source_distinct_id"); - expect(sql.leftJoinClause).toContain("version DESC"); - expect(sql.leftJoinClause).toContain("changed_at DESC"); - }); - - it("excludes latest tombstones from query-time resolution", () => { - const sql = buildResolvedEventsSql(); - - expect(sql.leftJoinClause).toContain("WHERE is_deleted = 0"); - }); -}); diff --git a/packages/clickhouse-db/src/analytics/resolved-events-sql.ts b/packages/clickhouse-db/src/analytics/resolved-events-sql.ts deleted file mode 100644 index 5cc2e1dda..000000000 --- a/packages/clickhouse-db/src/analytics/resolved-events-sql.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, -} from "./schema.ts"; - -export interface ResolvedEventsSqlParts { - readonly effectivePersonIdExpression: string; - readonly effectiveDistinctIdExpression: string; - readonly fromClause: string; - readonly leftJoinClause: string; -} - -export interface BuildResolvedEventsSqlOptions { - readonly eventsAlias?: string; - readonly overridesAlias?: string; -} - -const buildLatestPendingOverridesSubquery = (overridesAlias: string) => - ` -( - SELECT - project_id, - source_distinct_id, - target_distinct_id, - person_id - FROM ( - SELECT - project_id, - source_distinct_id, - target_distinct_id, - person_id, - is_deleted, - version, - changed_at - FROM ${CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE} - WHERE version > 0 - ORDER BY - project_id ASC, - source_distinct_id ASC, - version DESC, - changed_at DESC - LIMIT 1 BY project_id, source_distinct_id - ) - WHERE is_deleted = 0 -) AS ${overridesAlias}`.trim(); - -export const buildResolvedEventsSql = ( - options: BuildResolvedEventsSqlOptions = {}, -): ResolvedEventsSqlParts => { - const eventsAlias = options.eventsAlias ?? "events"; - const overridesAlias = options.overridesAlias ?? "pending_overrides"; - - return { - effectivePersonIdExpression: `coalesce(${overridesAlias}.person_id, ${eventsAlias}.person_id)`, - effectiveDistinctIdExpression: `coalesce(${overridesAlias}.target_distinct_id, ${eventsAlias}.distinct_id)`, - fromClause: `FROM ${CLICKHOUSE_EVENTS_TABLE} AS ${eventsAlias}`, - leftJoinClause: `LEFT JOIN ${buildLatestPendingOverridesSubquery(overridesAlias)} -ON ${overridesAlias}.project_id = ${eventsAlias}.project_id -AND ${overridesAlias}.source_distinct_id = ${eventsAlias}.distinct_id`, - }; -}; diff --git a/packages/clickhouse-db/src/analytics/schema.ts b/packages/clickhouse-db/src/analytics/schema.ts deleted file mode 100644 index 701f78718..000000000 --- a/packages/clickhouse-db/src/analytics/schema.ts +++ /dev/null @@ -1,10 +0,0 @@ -// ClickHouse analytics tables. Names are unqualified — the runtime client -// connects with a per-stage database (provisioned by `Clickhouse.Database`) -// as its default, so unqualified table references resolve into the correct -// per-stage database without hardcoding the database name here. -export const CLICKHOUSE_EVENTS_TABLE = "events_v2"; -export const CLICKHOUSE_PERSONS_TABLE = "persons_v1"; -export const CLICKHOUSE_PERSON_IDENTITY_TABLE = "person_identity_v1"; -export const CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE = "person_identity_overrides_v1"; -export const CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE = - "person_identity_pending_overrides_v2"; diff --git a/packages/clickhouse-db/src/clickhouse-client-web/ClickhouseWebClient.ts b/packages/clickhouse-db/src/clickhouse-client-web/ClickhouseWebClient.ts deleted file mode 100644 index 8d60324a5..000000000 --- a/packages/clickhouse-db/src/clickhouse-client-web/ClickhouseWebClient.ts +++ /dev/null @@ -1,965 +0,0 @@ -/** - * ClickHouse client implementation for Effect SQL, backed by - * `@clickhouse/client-web`. - * - * This module mirrors `@effect/sql-clickhouse`'s `ClickhouseClient` surface - * while avoiding Node-only APIs so it can run in Cloudflare Workers. - */ -import * as Clickhouse from "@clickhouse/client-web"; -import * as Clock from "effect/Clock"; -import * as Config from "effect/Config"; -import * as Context from "effect/Context"; -import * as Data from "effect/Data"; -import * as Duration from "effect/Duration"; -import * as Effect from "effect/Effect"; -import { dual } from "effect/Function"; -import * as Layer from "effect/Layer"; -import * as Random from "effect/Random"; -import type * as Scope from "effect/Scope"; -import * as Stream from "effect/Stream"; -import * as Reactivity from "effect/unstable/reactivity/Reactivity"; -import * as Client from "effect/unstable/sql/SqlClient"; -import type { Connection } from "effect/unstable/sql/SqlConnection"; -import { - AuthenticationError, - AuthorizationError, - ConnectionError, - SqlError, - SqlSyntaxError, - StatementTimeoutError, - UnknownError, -} from "effect/unstable/sql/SqlError"; -import * as Statement from "effect/unstable/sql/Statement"; - -const ATTR_DB_SYSTEM_NAME = "db.system.name"; -const ATTR_DB_NAMESPACE = "db.namespace"; - -const clickhouseCodeFromCause = (cause: unknown): number | undefined => { - if (typeof cause !== "object" || cause === null || !("code" in cause)) { - return undefined; - } - const code = cause.code; - if (typeof code === "number") { - return code; - } - if (typeof code === "string") { - const parsed = Number(code); - if (Number.isNaN(parsed)) { - return undefined; - } - return parsed; - } - return undefined; -}; - -const clickhouseSyntaxErrorCodes = new Set([36, 60, 62, 242]); - -const trimmedOrUndefined = (value: string): string | undefined => { - const trimmed = value.trim(); - if (trimmed.length > 0) { - return trimmed; - } - return undefined; -}; - -const messageFromCause = (cause: unknown): string | undefined => { - if (typeof cause === "string") { - return trimmedOrUndefined(cause); - } - if (typeof cause === "object" && cause !== null && "message" in cause) { - const message = cause.message; - if (typeof message === "string") { - return trimmedOrUndefined(message); - } - } - return undefined; -}; - -const withCauseMessage = (message: string, cause: unknown): string => { - const causeMessage = messageFromCause(cause); - if (causeMessage !== undefined && causeMessage !== message) { - return `${message}: ${causeMessage}`; - } - return message; -}; - -const classifyError = ( - cause: unknown, - message: string, - operation: string, - fallback: "connection" | "unknown" = "unknown", -) => { - const props = { cause, message: withCauseMessage(message, cause), operation }; - const code = clickhouseCodeFromCause(cause); - if (code !== undefined) { - if (code === 516) { - return new AuthenticationError(props); - } - if (code === 497) { - return new AuthorizationError(props); - } - if (clickhouseSyntaxErrorCodes.has(code)) { - return new SqlSyntaxError(props); - } - if (code === 159 || code === 469) { - return new StatementTimeoutError(props); - } - } - if (fallback === "connection") { - return new ConnectionError(props); - } - return new UnknownError(props); -}; - -/** - * Defect carried as the `cause` of a connection timeout, replacing a bare - * `Error` so the failure is a tagged value. - */ -class ConnectionTimeout extends Data.TaggedError("ClickhouseWebConnectionTimeout")<{ - readonly message: string; -}> {} - -const makeQueryId = Effect.gen(function* () { - const uuid = globalThis.crypto?.randomUUID?.(); - if (uuid !== undefined) { - return uuid; - } - const millis = yield* Clock.currentTimeMillis; - const random = yield* Random.next; - return `clickhouse-web-${millis}-${random.toString(36).slice(2)}`; -}); - -const resolveQueryId = (queryId: string | undefined): Effect.Effect => { - if (queryId !== undefined) { - return Effect.succeed(queryId); - } - return makeQueryId; -}; - -/** - * Build the per-request HTTP headers that carry a ClickHouse quota key. ClickHouse - * reads the quota bucket from the `X-ClickHouse-Quota` header, so a `KEYED BY - * client_key` quota only isolates principals when this is set per request; - * `undefined` (the default) sends no override and falls back to the empty key. - */ -const quotaHeaders = (quotaKey: string | undefined): Record | undefined => { - if (quotaKey) { - return { "X-ClickHouse-Quota": quotaKey }; - } - return undefined; -}; - -/** - * Unwraps the `data` envelope that JSON-shaped ClickHouse formats wrap rows in, - * while passing row arrays (`JSONEachRow` and friends) straight through. The - * parameter is `any` because the shape depends on the runtime format, which the - * `@clickhouse/client-web` types express as a single opaque union. - */ -const rowsFromJson = (value: any): ReadonlyArray => { - if (value !== null && typeof value === "object" && "data" in value) { - return value.data; - } - return value; -}; - -/** - * Adapts a `ResultSet` stream to the async-iterable shape `Stream.fromAsyncIterable` - * expects. `@clickhouse/client-web` types the stream as a `ReadableStream` even - * though it is async-iterable at runtime, so this is a typed boundary helper. - */ -const asyncRows = ( - stream: any, -): AsyncIterable>> => stream; - -const makeRowTransform = (transformResultNames: ((str: string) => string) | undefined) => { - if (transformResultNames === undefined) { - return undefined; - } - return Statement.defaultTransforms(transformResultNames).array; -}; - -const spanAttributeEntries = ( - attributes: Record | undefined, -): Array<[string, unknown]> => { - if (attributes === undefined) { - return []; - } - return Object.entries(attributes); -}; - -type WebInsertValues = - | ReadonlyArray - | Clickhouse.InputJSON - | Clickhouse.InputJSONObjectEachRow; - -/** - * Unique runtime identifier used to tag `ClickhouseWebClient` values. - */ -export const TypeId: TypeId = "~@voidhash/clickhouse-db/ClickhouseWebClient"; - -/** - * Type-level literal for the `ClickhouseWebClient` runtime identifier. - */ -export type TypeId = "~@voidhash/clickhouse-db/ClickhouseWebClient"; - -/** - * ClickHouse-specific `SqlClient` extension backed by `@clickhouse/client-web`. - */ -export interface ClickhouseWebClient extends Client.SqlClient { - readonly [TypeId]: TypeId; - readonly config: ClickhouseWebClientConfig; - readonly param: (dataType: string, value: unknown) => Statement.Fragment; - readonly asCommand: (effect: Effect.Effect) => Effect.Effect; - readonly insertQuery: (options: { - readonly table: string; - readonly values: WebInsertValues; - readonly format?: Clickhouse.DataFormat; - }) => Effect.Effect; - readonly withQueryId: { - (queryId: string): (effect: Effect.Effect) => Effect.Effect; - (effect: Effect.Effect, queryId: string): Effect.Effect; - }; - /** - * Tag the wrapped statements with a ClickHouse quota key (sent as the - * `X-ClickHouse-Quota` header). A `KEYED BY client_key` quota only isolates - * principals when each request carries a stable per-principal key. - */ - readonly withQuotaKey: { - (quotaKey: string): (effect: Effect.Effect) => Effect.Effect; - (effect: Effect.Effect, quotaKey: string): Effect.Effect; - }; - readonly withClickhouseSettings: { - ( - settings: NonNullable, - ): (effect: Effect.Effect) => Effect.Effect; - ( - effect: Effect.Effect, - settings: NonNullable, - ): Effect.Effect; - }; -} - -/** - * Context service tag for accessing the active `ClickhouseWebClient`. - */ -export const ClickhouseWebClient = Context.Service( - "@voidhash/clickhouse-db/ClickhouseWebClient", -); - -/** - * Configuration for creating a web ClickHouse client. - */ -export interface ClickhouseWebClientConfig extends Clickhouse.ClickHouseClientConfigOptions { - readonly spanAttributes?: Record | undefined; - readonly transformResultNames?: ((str: string) => string) | undefined; - readonly transformQueryNames?: ((str: string) => string) | undefined; -} - -/** - * Creates a scoped `ClickhouseWebClient` and verifies connectivity. - */ -export const make = ( - options: ClickhouseWebClientConfig, -): Effect.Effect => - Effect.gen(function* () { - const compiler = makeCompiler(options.transformQueryNames); - const transformRows = makeRowTransform(options.transformResultNames); - - const client = Clickhouse.createClient(options); - - const connectError = (cause: unknown) => - new SqlError({ - reason: classifyError( - cause, - "ClickhouseWebClient: Failed to connect", - "connect", - "connection", - ), - }); - - yield* Effect.acquireRelease( - Effect.tryPromise({ - try: () => client.ping(), - catch: connectError, - }).pipe( - Effect.flatMap((result) => { - if (result.success) { - return Effect.void; - } - return Effect.fail(connectError(result.error)); - }), - ), - () => Effect.promise(() => client.close()), - ).pipe( - Effect.timeoutOrElse({ - duration: Duration.seconds(5), - orElse: () => - Effect.fail( - new SqlError({ - reason: new ConnectionError({ - message: "ClickhouseWebClient: Connection timeout", - cause: new ConnectionTimeout({ message: "connection timeout" }), - operation: "connect", - }), - }), - ), - }), - ); - - class ConnectionImpl implements Connection { - private conn: Clickhouse.ClickHouseClient; - - constructor(conn: Clickhouse.ClickHouseClient) { - this.conn = conn; - } - - private runRaw( - sql: string, - params: ReadonlyArray, - format: Clickhouse.DataFormat = "JSON", - ) { - const paramsObj: Record = {}; - for (let i = 0; i < params.length; i++) { - paramsObj[`p${i + 1}`] = params[i]; - } - return Effect.withFiber< - Clickhouse.ResultSet | Clickhouse.CommandResult, - SqlError - >((fiber) => { - const method = fiber.getRef(ClientMethod); - return Effect.flatMap(resolveQueryId(fiber.getRef(QueryId)), (queryId) => - Effect.callback< - Clickhouse.ResultSet | Clickhouse.CommandResult, - SqlError - >((resume) => { - const settings = fiber.getRef(ClickhouseSettings); - const controller = new AbortController(); - if (method === "command") { - this.conn - .command({ - query: sql, - query_params: paramsObj, - abort_signal: controller.signal, - query_id: queryId, - clickhouse_settings: settings, - http_headers: quotaHeaders(fiber.getRef(QuotaKey)), - }) - .then( - (result) => resume(Effect.succeed(result)), - (cause) => - resume( - Effect.fail( - new SqlError({ - reason: classifyError(cause, "Failed to execute statement", "execute"), - }), - ), - ), - ); - } else { - this.conn - .query({ - query: sql, - query_params: paramsObj, - abort_signal: controller.signal, - query_id: queryId, - clickhouse_settings: settings, - http_headers: quotaHeaders(fiber.getRef(QuotaKey)), - format, - }) - .then( - (result) => resume(Effect.succeed(result)), - (cause) => - resume( - Effect.fail( - new SqlError({ - reason: classifyError(cause, "Failed to execute statement", "execute"), - }), - ), - ), - ); - } - return Effect.suspend(() => { - controller.abort(); - return Effect.promise(() => - this.conn.command({ query: `KILL QUERY WHERE query_id = '${queryId}'` }), - ); - }); - }), - ); - }); - } - - private run(sql: string, params: ReadonlyArray, format?: Clickhouse.DataFormat) { - return this.runRaw(sql, params, format).pipe( - Effect.flatMap((result) => { - if ("json" in result) { - return Effect.promise(() => result.json().then(rowsFromJson, () => [])); - } - return Effect.succeed([]); - }), - ); - } - - execute( - sql: string, - params: ReadonlyArray, - transformRows: ((row: ReadonlyArray) => ReadonlyArray) | undefined, - ) { - if (transformRows) { - return Effect.map(this.run(sql, params), transformRows); - } - return this.run(sql, params); - } - - executeRaw(sql: string, params: ReadonlyArray) { - return this.runRaw(sql, params); - } - - executeValues(sql: string, params: ReadonlyArray) { - return this.run(sql, params, "JSONCompact"); - } - - executeValuesUnprepared(sql: string, params: ReadonlyArray) { - return this.run(sql, params, "JSONCompact"); - } - - executeUnprepared(sql: string, params: ReadonlyArray, transformRows?: any) { - return this.execute(sql, params, transformRows); - } - - executeStream( - sql: string, - params: ReadonlyArray, - transformRows: ((row: ReadonlyArray) => ReadonlyArray) | undefined, - ) { - return this.runRaw(sql, params, "JSONEachRow").pipe( - Effect.map((result) => { - if (!("stream" in result)) { - return Stream.empty; - } - return Stream.fromAsyncIterable( - asyncRows(result.stream()), - (cause) => - new SqlError({ - reason: classifyError(cause, "Failed to execute stream", "stream"), - }), - ); - }), - Stream.unwrap, - Stream.mapEffect((rows) => - Effect.try({ - try: () => { - const parsed = rows.map((row) => row.json()); - if (transformRows) { - return transformRows(parsed); - } - return parsed; - }, - catch: (cause) => - new SqlError({ - reason: classifyError(cause, "Failed to parse row", "parseRow"), - }), - }), - ), - Stream.flattenIterable, - ); - } - } - - const connection = new ConnectionImpl(client); - - return Object.assign( - yield* Client.make({ - acquirer: Effect.succeed(connection), - compiler, - spanAttributes: [ - ...spanAttributeEntries(options.spanAttributes), - [ATTR_DB_SYSTEM_NAME, "clickhouse"], - [ATTR_DB_NAMESPACE, options.database ?? "default"], - ], - beginTransaction: "BEGIN TRANSACTION", - transformRows, - }), - { - [TypeId]: TypeId, - config: options, - param(dataType: string, value: unknown) { - return Statement.fragment([clickhouseParam(dataType, value)]); - }, - asCommand(effect: Effect.Effect) { - return Effect.provideService(effect, ClientMethod, "command"); - }, - insertQuery(options: { - readonly table: string; - readonly values: WebInsertValues; - readonly format?: Clickhouse.DataFormat; - }) { - return Effect.withFiber((fiber) => - Effect.flatMap(resolveQueryId(fiber.getRef(QueryId)), (queryId) => - Effect.callback((resume) => { - const settings = fiber.getRef(ClickhouseSettings); - const controller = new AbortController(); - client - .insert({ - format: "JSONEachRow", - ...options, - abort_signal: controller.signal, - query_id: queryId, - clickhouse_settings: settings, - }) - .then( - (result) => resume(Effect.succeed(result)), - (cause) => - resume( - Effect.fail( - new SqlError({ - reason: classifyError(cause, "Failed to insert data", "insert"), - }), - ), - ), - ); - return Effect.suspend(() => { - controller.abort(); - return Effect.promise(() => - client.command({ query: `KILL QUERY WHERE query_id = '${queryId}'` }), - ); - }); - }), - ), - ); - }, - withQueryId: dual(2, (effect: Effect.Effect, queryId: string) => - Effect.provideService(effect, QueryId, queryId), - ), - withQuotaKey: dual(2, (effect: Effect.Effect, quotaKey: string) => - Effect.provideService(effect, QuotaKey, quotaKey), - ), - withClickhouseSettings: dual( - 2, - ( - effect: Effect.Effect, - settings: NonNullable, - ) => Effect.provideService(effect, ClickhouseSettings, settings), - ), - }, - ); - }); - -/** - * Like {@link make}, but builds the underlying `@clickhouse/client-web` client - * lazily on first statement and performs NO connectivity check (`ping`). - * - * This makes a `ClickhouseWebClient` value safe to materialize eagerly even - * when the connection vars are not yet available — e.g. during Alchemy's plan - * phase, where a Worker's `env` bindings are still empty. `createClient` is - * cheap and connectionless, so deferring both it and the `getConfig` read to - * the first query means no network call happens until runtime. Because nothing - * is eagerly acquired it needs no `Scope` (only `Reactivity`); the web client is - * HTTP/stateless and requires no explicit close. - * - * `getConfig` is a thunk so the connection vars can be read from the Worker - * environment at first use rather than at construction time. It is invoked at - * most once (memoised), and its result is also exposed via the returned - * client's `config` getter. - */ -export const makeUnchecked = ( - getConfig: () => ClickhouseWebClientConfig, -): Effect.Effect => - Effect.gen(function* () { - let resolvedConfig: ClickhouseWebClientConfig | undefined; - const configOnce = () => (resolvedConfig ??= getConfig()); - let clientRef: Clickhouse.ClickHouseClient | undefined; - const clientOnce = () => (clientRef ??= Clickhouse.createClient(configOnce())); - - const compiler = makeCompiler(); - - class ConnectionImpl implements Connection { - private runRaw( - sql: string, - params: ReadonlyArray, - format: Clickhouse.DataFormat = "JSON", - ) { - const paramsObj: Record = {}; - for (let i = 0; i < params.length; i++) { - paramsObj[`p${i + 1}`] = params[i]; - } - return Effect.withFiber< - Clickhouse.ResultSet | Clickhouse.CommandResult, - SqlError - >((fiber) => { - const method = fiber.getRef(ClientMethod); - return Effect.flatMap(resolveQueryId(fiber.getRef(QueryId)), (queryId) => - Effect.callback< - Clickhouse.ResultSet | Clickhouse.CommandResult, - SqlError - >((resume) => { - const conn = clientOnce(); - const settings = fiber.getRef(ClickhouseSettings); - const controller = new AbortController(); - if (method === "command") { - conn - .command({ - query: sql, - query_params: paramsObj, - abort_signal: controller.signal, - query_id: queryId, - clickhouse_settings: settings, - http_headers: quotaHeaders(fiber.getRef(QuotaKey)), - }) - .then( - (result) => resume(Effect.succeed(result)), - (cause) => - resume( - Effect.fail( - new SqlError({ - reason: classifyError(cause, "Failed to execute statement", "execute"), - }), - ), - ), - ); - } else { - conn - .query({ - query: sql, - query_params: paramsObj, - abort_signal: controller.signal, - query_id: queryId, - clickhouse_settings: settings, - http_headers: quotaHeaders(fiber.getRef(QuotaKey)), - format, - }) - .then( - (result) => resume(Effect.succeed(result)), - (cause) => - resume( - Effect.fail( - new SqlError({ - reason: classifyError(cause, "Failed to execute statement", "execute"), - }), - ), - ), - ); - } - return Effect.suspend(() => { - controller.abort(); - return Effect.promise(() => - clientOnce().command({ query: `KILL QUERY WHERE query_id = '${queryId}'` }), - ); - }); - }), - ); - }); - } - - private run(sql: string, params: ReadonlyArray, format?: Clickhouse.DataFormat) { - return this.runRaw(sql, params, format).pipe( - Effect.flatMap((result) => { - if ("json" in result) { - return Effect.promise(() => result.json().then(rowsFromJson, () => [])); - } - return Effect.succeed([]); - }), - ); - } - - execute( - sql: string, - params: ReadonlyArray, - transformRows: ((row: ReadonlyArray) => ReadonlyArray) | undefined, - ) { - if (transformRows) { - return Effect.map(this.run(sql, params), transformRows); - } - return this.run(sql, params); - } - - executeRaw(sql: string, params: ReadonlyArray) { - return this.runRaw(sql, params); - } - - executeValues(sql: string, params: ReadonlyArray) { - return this.run(sql, params, "JSONCompact"); - } - - executeValuesUnprepared(sql: string, params: ReadonlyArray) { - return this.run(sql, params, "JSONCompact"); - } - - executeUnprepared(sql: string, params: ReadonlyArray, transformRows?: any) { - return this.execute(sql, params, transformRows); - } - - executeStream( - sql: string, - params: ReadonlyArray, - transformRows: ((row: ReadonlyArray) => ReadonlyArray) | undefined, - ) { - return this.runRaw(sql, params, "JSONEachRow").pipe( - Effect.map((result) => { - if (!("stream" in result)) { - return Stream.empty; - } - return Stream.fromAsyncIterable( - asyncRows(result.stream()), - (cause) => - new SqlError({ - reason: classifyError(cause, "Failed to execute stream", "stream"), - }), - ); - }), - Stream.unwrap, - Stream.mapEffect((rows) => - Effect.try({ - try: () => { - const parsed = rows.map((row) => row.json()); - if (transformRows) { - return transformRows(parsed); - } - return parsed; - }, - catch: (cause) => - new SqlError({ - reason: classifyError(cause, "Failed to parse row", "parseRow"), - }), - }), - ), - Stream.flattenIterable, - ); - } - } - - const connection = new ConnectionImpl(); - - const client = Object.assign( - yield* Client.make({ - acquirer: Effect.succeed(connection), - compiler, - spanAttributes: [ - [ATTR_DB_SYSTEM_NAME, "clickhouse"], - [ATTR_DB_NAMESPACE, "default"], - ], - beginTransaction: "BEGIN TRANSACTION", - }), - { - [TypeId]: TypeId, - param(dataType: string, value: unknown) { - return Statement.fragment([clickhouseParam(dataType, value)]); - }, - asCommand(effect: Effect.Effect) { - return Effect.provideService(effect, ClientMethod, "command"); - }, - insertQuery(options: { - readonly table: string; - readonly values: WebInsertValues; - readonly format?: Clickhouse.DataFormat; - }) { - return Effect.withFiber((fiber) => - Effect.flatMap(resolveQueryId(fiber.getRef(QueryId)), (queryId) => - Effect.callback((resume) => { - const conn = clientOnce(); - const settings = fiber.getRef(ClickhouseSettings); - const controller = new AbortController(); - conn - .insert({ - format: "JSONEachRow", - ...options, - abort_signal: controller.signal, - query_id: queryId, - clickhouse_settings: settings, - }) - .then( - (result) => resume(Effect.succeed(result)), - (cause) => - resume( - Effect.fail( - new SqlError({ - reason: classifyError(cause, "Failed to insert data", "insert"), - }), - ), - ), - ); - return Effect.suspend(() => { - controller.abort(); - return Effect.promise(() => - clientOnce().command({ query: `KILL QUERY WHERE query_id = '${queryId}'` }), - ); - }); - }), - ), - ); - }, - withQueryId: dual(2, (effect: Effect.Effect, queryId: string) => - Effect.provideService(effect, QueryId, queryId), - ), - withQuotaKey: dual(2, (effect: Effect.Effect, quotaKey: string) => - Effect.provideService(effect, QuotaKey, quotaKey), - ), - withClickhouseSettings: dual( - 2, - ( - effect: Effect.Effect, - settings: NonNullable, - ) => Effect.provideService(effect, ClickhouseSettings, settings), - ), - }, - ); - - // `config` is exposed lazily so reading it never forces the `getConfig` - // thunk (and thus the Worker env read) before the first real use. - return withLazyConfig(client, configOnce); - }); - -/** - * Installs the lazy `config` getter that completes a `ClickhouseWebClient`. - * - * The client argument is `any` because `Object.defineProperty` cannot express - * the added property in the type system; this helper is the single place where - * that gap is bridged. - */ -const withLazyConfig = ( - client: any, - getConfig: () => ClickhouseWebClientConfig, -): ClickhouseWebClient => { - Object.defineProperty(client, "config", { get: getConfig, enumerable: true }); - return client; -}; - -/** - * Fiber reference read by the low-level ClickHouse connection to choose query - * or command execution for statements; defaults to `query`. - */ -export const ClientMethod = Context.Reference<"query" | "command" | "insert">( - "@voidhash/clickhouse-db/ClickhouseWebClient/ClientMethod", - { - defaultValue: () => "query", - }, -); - -/** - * Fiber reference for the ClickHouse `query_id` applied to queries and inserts. - */ -export const QueryId = Context.Reference( - "@voidhash/clickhouse-db/ClickhouseWebClient/QueryId", - { defaultValue: () => undefined }, -); - -/** - * Fiber reference for the ClickHouse quota key (`X-ClickHouse-Quota` header) - * applied to queries, commands, and inserts. Defaults to `undefined` (no header, - * empty-key bucket) so existing callers are unaffected. - */ -export const QuotaKey = Context.Reference( - "@voidhash/clickhouse-db/ClickhouseWebClient/QuotaKey", - { defaultValue: () => undefined }, -); - -/** - * Fiber reference containing ClickHouse settings to attach to queries, - * commands, and inserts. - */ -export const ClickhouseSettings: Context.Reference< - NonNullable -> = Context.Reference("@voidhash/clickhouse-db/ClickhouseWebClient/ClickhouseSettings", { - defaultValue: () => ({}), -}); - -/** - * Provides both `ClickhouseWebClient` and generic `SqlClient` services from a - * `Config`-backed ClickHouse client configuration. - */ -export const layerConfig: ( - config: Config.Wrap, -) => Layer.Layer = ( - config: Config.Wrap, -): Layer.Layer => - Layer.effectContext( - Config.unwrap(config).pipe( - Effect.flatMap(make), - Effect.map((client) => - Context.make(ClickhouseWebClient, client).pipe(Context.add(Client.SqlClient, client)), - ), - ), - ).pipe(Layer.provide(Reactivity.layer)); - -/** - * Provides both `ClickhouseWebClient` and generic `SqlClient` services from a - * ClickHouse client configuration. - */ -export const layer = ( - config: ClickhouseWebClientConfig, -): Layer.Layer => - Layer.effectContext( - Effect.map(make(config), (client) => - Context.make(ClickhouseWebClient, client).pipe(Context.add(Client.SqlClient, client)), - ), - ).pipe(Layer.provide(Reactivity.layer)); - -const typeFromUnknown = (value: unknown): string => { - if (Statement.isFragment(value)) { - return typeFromUnknown(value.segments[0]); - } else if (isClickhouseParam(value)) { - return value.paramA; - } else if (Array.isArray(value)) { - return `Array(${typeFromUnknown(value[0])})`; - } - switch (typeof value) { - case "number": - return "Decimal"; - case "bigint": - return "Int64"; - case "boolean": - return "Bool"; - case "object": - if (value instanceof Date) { - return "DateTime()"; - } - return "String"; - default: - return "String"; - } -}; - -/** - * Creates the SQL statement compiler for ClickHouse. - */ -export const makeCompiler = (transform?: (_: string) => string) => - Statement.makeCompiler({ - dialect: "sqlite", - placeholder(i, u) { - return `{p${i}: ${typeFromUnknown(u)}}`; - }, - onIdentifier: makeOnIdentifier(transform), - onRecordUpdate() { - return ["", []]; - }, - onCustom(type, placeholder) { - return [placeholder(type), [type.paramB]]; - }, - }); - -const escape = Statement.defaultEscape('"'); - -const makeOnIdentifier = ( - transform: ((_: string) => string) | undefined, -): ((value: string, withoutTransform: boolean) => string) => { - if (transform === undefined) { - return escape; - } - return (value, withoutTransform) => { - if (withoutTransform) { - return escape(value); - } - return escape(transform(value)); - }; -}; - -/** - * Custom SQL fragment type used for ClickHouse typed parameters created by - * `ClickhouseWebClient.param`. - */ -export type ClickhouseCustom = ClickhouseParam; - -interface ClickhouseParam extends Statement.Custom<"ClickhouseParam", string, unknown> {} - -const clickhouseParam = Statement.custom("ClickhouseParam"); -const isClickhouseParam = Statement.isCustom("ClickhouseParam"); diff --git a/packages/clickhouse-db/src/clickhouse-client-web/index.ts b/packages/clickhouse-db/src/clickhouse-client-web/index.ts deleted file mode 100644 index d84f2f70e..000000000 --- a/packages/clickhouse-db/src/clickhouse-client-web/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Worker-safe ClickHouse client for Effect SQL. - */ -export * as ClickhouseWebClient from "./ClickhouseWebClient.ts"; diff --git a/packages/clickhouse-db/src/index.ts b/packages/clickhouse-db/src/index.ts deleted file mode 100644 index 835f87086..000000000 --- a/packages/clickhouse-db/src/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from "./live.ts"; -export { ClickhouseWebClient } from "./clickhouse-client-web/index.ts"; diff --git a/packages/clickhouse-db/src/live.ts b/packages/clickhouse-db/src/live.ts deleted file mode 100644 index 864db308d..000000000 --- a/packages/clickhouse-db/src/live.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { Effect, Layer } from "effect"; -import { SqlClient, type SqlError } from "effect/unstable/sql"; - -import { ClickhouseWebClient } from "./clickhouse-client-web/index.ts"; - -export type ClickhouseDbConfig = { - readonly database?: string; - readonly url: string; - readonly username?: string; - readonly password?: string; -}; - -export type Migration = readonly [ - id: number, - name: string, - effect: Effect.Effect< - void, - SqlError.SqlError, - SqlClient.SqlClient | ClickhouseWebClient.ClickhouseWebClient - >, -]; - -export type MigrationSet = { - readonly tableName: string; - readonly migrations: ReadonlyArray; -}; - -/** - * Reads the highest applied migration id from the ledger query result, or `-1` - * when the ledger is empty (or the row does not carry a numeric id). - */ -const latestMigrationId = (rows: ReadonlyArray): number => { - const row = rows[0]; - if (typeof row === "object" && row !== null && "migration_id" in row) { - const id = row.migration_id; - if (typeof id === "number") { - return id; - } - } - return -1; -}; - -const runMigrationSet = (migrationSet: MigrationSet) => - Layer.effectDiscard( - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - - yield* ch.asCommand(sql` - CREATE TABLE IF NOT EXISTS ${sql(migrationSet.tableName)} - ( - migration_id Int32, - created_at DateTime DEFAULT now(), - name String - ) - ENGINE = MergeTree - ORDER BY migration_id - `); - - const applied = yield* sql` - SELECT migration_id FROM ${sql(migrationSet.tableName)} ORDER BY migration_id DESC LIMIT 1 - `.withoutTransform; - const latestId = latestMigrationId(applied); - - for (const [id, name, migration] of migrationSet.migrations) { - if (id <= latestId) continue; - yield* migration; - yield* ch.asCommand(sql` - INSERT INTO ${sql(migrationSet.tableName)} (migration_id, name) - VALUES (${id}, ${name}) - `); - } - }), - ); - -/** Builds a scoped ClickHouse client and applies the requested migration sets. */ -export const ClickhouseDbLive = ( - { database, url, username, password }: ClickhouseDbConfig, - migrationSets: ReadonlyArray = [], -) => { - const clientLayer = ClickhouseWebClient.layer({ - database, - password, - url, - username, - }); - - if (migrationSets.length === 0) { - return clientLayer; - } - - const migrationLayers = migrationSets.map(runMigrationSet); - const allMigrations = migrationLayers.reduce((acc, layer) => acc.pipe(Layer.merge(layer))); - - return allMigrations.pipe(Layer.provideMerge(clientLayer)); -}; diff --git a/packages/clickhouse-db/src/migrations/0001_create_analytics_events.ts b/packages/clickhouse-db/src/migrations/0001_create_analytics_events.ts deleted file mode 100644 index 47ce6356d..000000000 --- a/packages/clickhouse-db/src/migrations/0001_create_analytics_events.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { CLICKHOUSE_EVENTS_TABLE } from "../analytics/schema.ts"; -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -export default Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - - yield* ch.asCommand(sql` - CREATE TABLE IF NOT EXISTS ${sql(CLICKHOUSE_EVENTS_TABLE)} - ( - event_id String, - event_name String, - - event_ts DateTime64(3), - ingestion_ts DateTime64(3), - - organization_id String, - project_id String, - - distinct_id String, - session_id Nullable(String), - - source LowCardinality(String), - - properties Map(String, String), - context Map(String, String), - - schema_version UInt8 - ) - ENGINE = MergeTree - PARTITION BY toYYYYMM(event_ts) - ORDER BY (organization_id, project_id, event_name, event_ts, event_id) - `); -}); diff --git a/packages/clickhouse-db/src/migrations/0002_create_analytics_identity_v2.test.ts b/packages/clickhouse-db/src/migrations/0002_create_analytics_identity_v2.test.ts deleted file mode 100644 index b51083bd1..000000000 --- a/packages/clickhouse-db/src/migrations/0002_create_analytics_identity_v2.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { migration0002Statements } from "./0002_create_analytics_identity_v2.ts"; - -describe("migration0002Statements", () => { - it("rebuilds processed-event ingestion objects before recreating the materialized view", () => { - const dropViewIndex = migration0002Statements.indexOf( - "DROP TABLE IF EXISTS event_processed_v2_mv", - ); - const dropKafkaIndex = migration0002Statements.indexOf( - "DROP TABLE IF EXISTS event_processed_v2_kafka", - ); - const dropEventsIndex = migration0002Statements.indexOf("DROP TABLE IF EXISTS events_v2"); - const createEventsIndex = migration0002Statements.findIndex((statement) => - statement.includes("CREATE TABLE events_v2"), - ); - const createKafkaIndex = migration0002Statements.findIndex((statement) => - statement.includes("CREATE TABLE IF NOT EXISTS event_processed_v2_kafka"), - ); - const createViewIndex = migration0002Statements.findIndex((statement) => - statement.includes("CREATE MATERIALIZED VIEW IF NOT EXISTS event_processed_v2_mv"), - ); - - expect(dropViewIndex).toBeGreaterThanOrEqual(0); - expect(dropKafkaIndex).toBe(dropViewIndex + 1); - expect(dropEventsIndex).toBe(dropKafkaIndex + 1); - expect(createEventsIndex).toBe(dropEventsIndex + 1); - expect(createKafkaIndex).toBeGreaterThan(createEventsIndex); - expect(createViewIndex).toBeGreaterThan(createKafkaIndex); - }); - - it("defines events_v2 with the processed-event columns expected by the materialized view", () => { - const createEventsTable = migration0002Statements.find((statement) => - statement.includes("CREATE TABLE events_v2"), - ); - - expect(createEventsTable).toBeDefined(); - expect(createEventsTable).toContain("event_id String"); - expect(createEventsTable).toContain("capture_id String"); - expect(createEventsTable).toContain("event_name String"); - expect(createEventsTable).toContain("event_ts DateTime64(3)"); - expect(createEventsTable).toContain("processed_ts DateTime64(3)"); - expect(createEventsTable).toContain("organization_id String"); - expect(createEventsTable).toContain("project_id String"); - expect(createEventsTable).toContain("distinct_id String"); - expect(createEventsTable).toContain("previous_distinct_id Nullable(String)"); - expect(createEventsTable).toContain("person_id Nullable(String)"); - expect(createEventsTable).toContain("identity_mode LowCardinality(String)"); - expect(createEventsTable).toContain("event_properties String"); - expect(createEventsTable).toContain("context String"); - expect(createEventsTable).toContain("route_lane LowCardinality(String)"); - expect(createEventsTable).toContain("skip_enrichment UInt8"); - expect(createEventsTable).toContain("source_offset String"); - expect(createEventsTable).toContain("source_partition Int32"); - expect(createEventsTable).toContain("source_topic String"); - expect(createEventsTable).toContain("token String"); - expect(createEventsTable).toContain("request_path LowCardinality(String)"); - expect(createEventsTable).toContain("request_id String"); - expect(createEventsTable).toContain("schema_version UInt8"); - }); - - it("keeps the materialized view aliases aligned with the rebuilt events_v2 schema", () => { - const createMaterializedView = migration0002Statements.find((statement) => - statement.includes("CREATE MATERIALIZED VIEW IF NOT EXISTS event_processed_v2_mv"), - ); - - expect(createMaterializedView).toBeDefined(); - expect(createMaterializedView).toContain( - "JSONExtractString(raw, 'processedEventId') AS event_id", - ); - expect(createMaterializedView).toContain("JSONExtractString(raw, 'captureId') AS capture_id"); - expect(createMaterializedView).toContain("JSONExtractString(raw, 'event') AS event_name"); - expect(createMaterializedView).toContain( - "parseDateTime64BestEffort(JSONExtractString(raw, 'processedAt'), 3) AS processed_ts", - ); - expect(createMaterializedView).toContain( - "nullIf(JSONExtractString(raw, 'previousDistinctId'), '') AS previous_distinct_id", - ); - expect(createMaterializedView).toContain( - "nullIf(JSONExtractString(raw, 'identity', 'personId'), '') AS person_id", - ); - expect(createMaterializedView).toContain( - "JSONExtractString(raw, 'identity', 'mode') AS identity_mode", - ); - expect(createMaterializedView).toContain( - "JSONExtractRaw(raw, 'properties') AS event_properties", - ); - expect(createMaterializedView).toContain( - "JSONExtractString(raw, 'routing', 'lane') AS route_lane", - ); - expect(createMaterializedView).toContain( - "toUInt8(JSONExtractBool(raw, 'routing', 'skipEnrichment')) AS skip_enrichment", - ); - expect(createMaterializedView).toContain( - "JSONExtractString(raw, 'routing', 'sourceOffset') AS source_offset", - ); - expect(createMaterializedView).toContain( - "toInt32(JSONExtractInt(raw, 'routing', 'sourcePartition')) AS source_partition", - ); - expect(createMaterializedView).toContain( - "JSONExtractString(raw, 'routing', 'sourceTopic') AS source_topic", - ); - expect(createMaterializedView).toContain("JSONExtractString(raw, 'token') AS token"); - expect(createMaterializedView).toContain( - "JSONExtractString(raw, 'request', 'path') AS request_path", - ); - expect(createMaterializedView).toContain( - "JSONExtractString(raw, 'request', 'requestId') AS request_id", - ); - expect(createMaterializedView).toContain( - "toUInt8(JSONExtractInt(raw, 'schemaVersion')) AS schema_version", - ); - }); -}); diff --git a/packages/clickhouse-db/src/migrations/0002_create_analytics_identity_v2.ts b/packages/clickhouse-db/src/migrations/0002_create_analytics_identity_v2.ts deleted file mode 100644 index 218dd0ceb..000000000 --- a/packages/clickhouse-db/src/migrations/0002_create_analytics_identity_v2.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { constant } from "@voidhash/lib/lang"; - -import { - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSONS_TABLE, -} from "../analytics/schema.ts"; -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -const KAFKA_BROKER_LIST = "redpanda:9092"; - -export const migration0002Statements = constant([ - `DROP TABLE IF EXISTS event_processed_v2_mv`, - `DROP TABLE IF EXISTS event_processed_v2_kafka`, - `DROP TABLE IF EXISTS ${CLICKHOUSE_EVENTS_TABLE}`, - `CREATE TABLE ${CLICKHOUSE_EVENTS_TABLE} - ( - event_id String, - capture_id String, - event_name String, - event_ts DateTime64(3), - processed_ts DateTime64(3), - organization_id String, - project_id String, - distinct_id String, - previous_distinct_id Nullable(String), - person_id Nullable(String), - identity_mode LowCardinality(String), - event_properties String, - context String, - route_lane LowCardinality(String), - skip_enrichment UInt8, - source_offset String, - source_partition Int32, - source_topic String, - token String, - request_path LowCardinality(String), - request_id String, - schema_version UInt8 - ) - ENGINE = MergeTree - PARTITION BY toYYYYMM(event_ts) - ORDER BY (project_id, distinct_id, event_ts, event_id)`, - `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_PERSONS_TABLE} - ( - person_id String, - project_id String, - primary_distinct_id Nullable(String), - email Nullable(String), - name Nullable(String), - traits String, - is_archived UInt8, - merged_into_person_id Nullable(String), - version UInt64, - changed_at DateTime64(3) - ) - ENGINE = ReplacingMergeTree(version) - PARTITION BY toYYYYMM(changed_at) - ORDER BY (project_id, person_id)`, - `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_PERSON_IDENTITY_TABLE} - ( - project_id String, - distinct_id String, - previous_distinct_id Nullable(String), - person_id String, - is_deleted UInt8, - version UInt64, - changed_at DateTime64(3) - ) - ENGINE = ReplacingMergeTree(version) - PARTITION BY toYYYYMM(changed_at) - ORDER BY (project_id, distinct_id)`, - `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE} - ( - project_id String, - distinct_id String, - previous_distinct_id Nullable(String), - person_id String, - is_deleted UInt8, - version UInt64, - changed_at DateTime64(3) - ) - ENGINE = ReplacingMergeTree(version) - PARTITION BY toYYYYMM(changed_at) - ORDER BY (project_id, distinct_id)`, - `CREATE TABLE IF NOT EXISTS event_processed_v2_kafka - ( - raw String - ) - ENGINE = Kafka - SETTINGS - kafka_broker_list = '${KAFKA_BROKER_LIST}', - kafka_topic_list = 'event.processed.v2', - kafka_group_name = 'clickhouse-event-processed-v2', - kafka_format = 'JSONAsString', - kafka_num_consumers = 1`, - `CREATE MATERIALIZED VIEW IF NOT EXISTS event_processed_v2_mv - TO ${CLICKHOUSE_EVENTS_TABLE} - AS - SELECT - JSONExtractString(raw, 'processedEventId') AS event_id, - JSONExtractString(raw, 'captureId') AS capture_id, - JSONExtractString(raw, 'event') AS event_name, - parseDateTime64BestEffort(JSONExtractString(raw, 'eventTimestamp'), 3) AS event_ts, - parseDateTime64BestEffort(JSONExtractString(raw, 'processedAt'), 3) AS processed_ts, - JSONExtractString(raw, 'organizationId') AS organization_id, - JSONExtractString(raw, 'projectId') AS project_id, - JSONExtractString(raw, 'identity', 'distinctId') AS distinct_id, - nullIf(JSONExtractString(raw, 'previousDistinctId'), '') AS previous_distinct_id, - nullIf(JSONExtractString(raw, 'identity', 'personId'), '') AS person_id, - JSONExtractString(raw, 'identity', 'mode') AS identity_mode, - JSONExtractRaw(raw, 'properties') AS event_properties, - JSONExtractRaw(raw, 'context') AS context, - JSONExtractString(raw, 'routing', 'lane') AS route_lane, - toUInt8(JSONExtractBool(raw, 'routing', 'skipEnrichment')) AS skip_enrichment, - JSONExtractString(raw, 'routing', 'sourceOffset') AS source_offset, - toInt32(JSONExtractInt(raw, 'routing', 'sourcePartition')) AS source_partition, - JSONExtractString(raw, 'routing', 'sourceTopic') AS source_topic, - JSONExtractString(raw, 'token') AS token, - JSONExtractString(raw, 'request', 'path') AS request_path, - JSONExtractString(raw, 'request', 'requestId') AS request_id, - toUInt8(JSONExtractInt(raw, 'schemaVersion')) AS schema_version - FROM event_processed_v2_kafka`, - `CREATE TABLE IF NOT EXISTS event_person_v1_kafka - ( - raw String - ) - ENGINE = Kafka - SETTINGS - kafka_broker_list = '${KAFKA_BROKER_LIST}', - kafka_topic_list = 'event.person.v1', - kafka_group_name = 'clickhouse-event-person-v1', - kafka_format = 'JSONAsString', - kafka_num_consumers = 1`, - `CREATE MATERIALIZED VIEW IF NOT EXISTS event_person_v1_mv - TO ${CLICKHOUSE_PERSONS_TABLE} - AS - SELECT - JSONExtractString(raw, 'personId') AS person_id, - JSONExtractString(raw, 'projectId') AS project_id, - nullIf(JSONExtractString(raw, 'primaryDistinctId'), '') AS primary_distinct_id, - nullIf(JSONExtractString(raw, 'email'), '') AS email, - nullIf(JSONExtractString(raw, 'name'), '') AS name, - JSONExtractRaw(raw, 'traits') AS traits, - toUInt8(JSONExtractBool(raw, 'isArchived')) AS is_archived, - nullIf(JSONExtractString(raw, 'mergedIntoPersonId'), '') AS merged_into_person_id, - toUInt64(JSONExtractInt(raw, 'version')) AS version, - parseDateTime64BestEffort(JSONExtractString(raw, 'changedAt'), 3) AS changed_at - FROM event_person_v1_kafka`, - `CREATE TABLE IF NOT EXISTS event_person_identity_v1_kafka - ( - raw String - ) - ENGINE = Kafka - SETTINGS - kafka_broker_list = '${KAFKA_BROKER_LIST}', - kafka_topic_list = 'event.person-distinct-id.v1', - kafka_group_name = 'clickhouse-event-person-identity-v1', - kafka_format = 'JSONAsString', - kafka_num_consumers = 1`, - `CREATE MATERIALIZED VIEW IF NOT EXISTS event_person_identity_v1_mv - TO ${CLICKHOUSE_PERSON_IDENTITY_TABLE} - AS - SELECT - JSONExtractString(raw, 'projectId') AS project_id, - JSONExtractString(raw, 'distinctId') AS distinct_id, - nullIf(JSONExtractString(raw, 'previousDistinctId'), '') AS previous_distinct_id, - JSONExtractString(raw, 'personId') AS person_id, - toUInt8(JSONExtractBool(raw, 'isDeleted')) AS is_deleted, - toUInt64(JSONExtractInt(raw, 'version')) AS version, - parseDateTime64BestEffort(JSONExtractString(raw, 'changedAt'), 3) AS changed_at - FROM event_person_identity_v1_kafka`, - `CREATE MATERIALIZED VIEW IF NOT EXISTS event_person_identity_overrides_v1_mv - TO ${CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE} - AS - SELECT - JSONExtractString(raw, 'projectId') AS project_id, - JSONExtractString(raw, 'distinctId') AS distinct_id, - nullIf(JSONExtractString(raw, 'previousDistinctId'), '') AS previous_distinct_id, - JSONExtractString(raw, 'personId') AS person_id, - toUInt8(JSONExtractBool(raw, 'isDeleted')) AS is_deleted, - toUInt64(JSONExtractInt(raw, 'version')) AS version, - parseDateTime64BestEffort(JSONExtractString(raw, 'changedAt'), 3) AS changed_at - FROM event_person_identity_v1_kafka - WHERE - toUInt64(JSONExtractInt(raw, 'version')) > 0 - AND nullIf(JSONExtractString(raw, 'previousDistinctId'), '') IS NOT NULL`, -]); - -export default Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - - for (const statement of migration0002Statements) { - yield* ch.asCommand(sql`${sql.literal(statement)}`); - } -}); diff --git a/packages/clickhouse-db/src/migrations/0003_create_analytics_identity_pending_overrides_v2.ts b/packages/clickhouse-db/src/migrations/0003_create_analytics_identity_pending_overrides_v2.ts deleted file mode 100644 index 222c17289..000000000 --- a/packages/clickhouse-db/src/migrations/0003_create_analytics_identity_pending_overrides_v2.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { constant } from "@voidhash/lib/lang"; - -import { - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, -} from "../analytics/schema.ts"; -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -const KAFKA_BROKER_LIST = "redpanda:9092"; - -const statements = constant([ - `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_PERSON_IDENTITY_TABLE} - ( - project_id String, - distinct_id String, - previous_distinct_id Nullable(String), - person_id String, - is_deleted UInt8, - version UInt64, - changed_at DateTime64(3) - ) - ENGINE = ReplacingMergeTree(version) - PARTITION BY toYYYYMM(changed_at) - ORDER BY (project_id, distinct_id)`, - `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE} - ( - project_id String, - distinct_id String, - previous_distinct_id Nullable(String), - person_id String, - is_deleted UInt8, - version UInt64, - changed_at DateTime64(3) - ) - ENGINE = ReplacingMergeTree(version) - PARTITION BY toYYYYMM(changed_at) - ORDER BY (project_id, distinct_id)`, - `CREATE TABLE IF NOT EXISTS event_person_identity_v1_kafka - ( - raw String - ) - ENGINE = Kafka - SETTINGS - kafka_broker_list = '${KAFKA_BROKER_LIST}', - kafka_topic_list = 'event.person-distinct-id.v1', - kafka_group_name = 'clickhouse-event-person-identity-v1', - kafka_format = 'JSONAsString', - kafka_num_consumers = 1`, - `CREATE MATERIALIZED VIEW IF NOT EXISTS event_person_identity_v1_mv - TO ${CLICKHOUSE_PERSON_IDENTITY_TABLE} - AS - SELECT - JSONExtractString(raw, 'projectId') AS project_id, - JSONExtractString(raw, 'distinctId') AS distinct_id, - nullIf(JSONExtractString(raw, 'previousDistinctId'), '') AS previous_distinct_id, - JSONExtractString(raw, 'personId') AS person_id, - toUInt8(JSONExtractBool(raw, 'isDeleted')) AS is_deleted, - toUInt64(JSONExtractInt(raw, 'version')) AS version, - parseDateTime64BestEffort(JSONExtractString(raw, 'changedAt'), 3) AS changed_at - FROM event_person_identity_v1_kafka`, - `CREATE MATERIALIZED VIEW IF NOT EXISTS event_person_identity_overrides_v1_mv - TO ${CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE} - AS - SELECT - JSONExtractString(raw, 'projectId') AS project_id, - JSONExtractString(raw, 'distinctId') AS distinct_id, - nullIf(JSONExtractString(raw, 'previousDistinctId'), '') AS previous_distinct_id, - JSONExtractString(raw, 'personId') AS person_id, - toUInt8(JSONExtractBool(raw, 'isDeleted')) AS is_deleted, - toUInt64(JSONExtractInt(raw, 'version')) AS version, - parseDateTime64BestEffort(JSONExtractString(raw, 'changedAt'), 3) AS changed_at - FROM event_person_identity_v1_kafka - WHERE - toUInt64(JSONExtractInt(raw, 'version')) > 0 - AND nullIf(JSONExtractString(raw, 'previousDistinctId'), '') IS NOT NULL`, - `CREATE TABLE IF NOT EXISTS ${CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE} - ( - project_id String, - source_distinct_id String, - target_distinct_id String, - person_id String, - is_deleted UInt8, - version UInt64, - changed_at DateTime64(3) - ) - ENGINE = ReplacingMergeTree(version) - PARTITION BY toYYYYMM(changed_at) - ORDER BY (project_id, source_distinct_id)`, - `CREATE MATERIALIZED VIEW IF NOT EXISTS event_person_identity_pending_overrides_v2_mv - TO ${CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE} - AS - SELECT - JSONExtractString(raw, 'projectId') AS project_id, - JSONExtractString(raw, 'previousDistinctId') AS source_distinct_id, - JSONExtractString(raw, 'distinctId') AS target_distinct_id, - JSONExtractString(raw, 'personId') AS person_id, - toUInt8(JSONExtractBool(raw, 'isDeleted')) AS is_deleted, - toUInt64(JSONExtractInt(raw, 'version')) AS version, - parseDateTime64BestEffort(JSONExtractString(raw, 'changedAt'), 3) AS changed_at - FROM event_person_identity_v1_kafka - WHERE - toUInt64(JSONExtractInt(raw, 'version')) > 0 - AND nullIf(JSONExtractString(raw, 'previousDistinctId'), '') IS NOT NULL`, - `INSERT INTO ${CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE} - ( - project_id, - source_distinct_id, - target_distinct_id, - person_id, - is_deleted, - version, - changed_at - ) - SELECT - project_id, - previous_distinct_id AS source_distinct_id, - distinct_id AS target_distinct_id, - person_id, - is_deleted, - version, - changed_at - FROM ${CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE}`, -]); - -export default Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - - for (const statement of statements) { - yield* ch.asCommand(sql`${sql.literal(statement)}`); - } -}); diff --git a/packages/clickhouse-db/src/migrations/0004_person_identity_cutover.ts b/packages/clickhouse-db/src/migrations/0004_person_identity_cutover.ts deleted file mode 100644 index 907ff1065..000000000 --- a/packages/clickhouse-db/src/migrations/0004_person_identity_cutover.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { constant } from "@voidhash/lib/lang"; - -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -const statements = constant([]); - -export default Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - - for (const statement of statements) { - yield* ch.asCommand(sql`${sql.literal(statement)}`); - } -}); diff --git a/packages/clickhouse-db/src/migrations/0005_align_analytics_identity_tables.ts b/packages/clickhouse-db/src/migrations/0005_align_analytics_identity_tables.ts deleted file mode 100644 index be25fcb83..000000000 --- a/packages/clickhouse-db/src/migrations/0005_align_analytics_identity_tables.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - CLICKHOUSE_PERSONS_TABLE, -} from "../analytics/schema.ts"; -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -interface DescribeColumnRow { - name: string; -} - -const getColumnNames = (table: string) => - Effect.gen(function* getColumnNamesEffect() { - const sql = yield* SqlClient.SqlClient; - const rows = yield* sql` - DESCRIBE TABLE ${sql.literal(table)} - `; - - return new Set(rows.map((row) => row.name)); - }); - -const execute = (statement: string) => - Effect.gen(function* executeEffect() { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - yield* ch.asCommand(sql`${sql.literal(statement)}`); - }); - -/** - * The expression a new column should be back-filled from, or `undefined` when - * the legacy column no longer exists on the table. - */ -const legacyColumnExpression = ( - columns: ReadonlySet, - column: string, - expression: string = column, -): string | undefined => { - if (!columns.has(column)) { - return undefined; - } - return expression; -}; - -const defaultClause = (legacyExpression: string | undefined): string => { - if (!legacyExpression) { - return ""; - } - return ` DEFAULT ${legacyExpression}`; -}; - -const addColumnFromLegacy = ({ - columns, - fallbackType, - legacyExpression, - name, - table, -}: { - columns: ReadonlySet; - fallbackType: string; - legacyExpression?: string; - name: string; - table: string; -}) => { - if (columns.has(name)) { - return Effect.void; - } - - const expression = defaultClause(legacyExpression); - return execute( - `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${name} ${fallbackType}${expression}`, - ); -}; - -export default Effect.gen(function* alignAnalyticsIdentityTables() { - const eventColumns = yield* getColumnNames(CLICKHOUSE_EVENTS_TABLE); - if (!eventColumns.has("previous_distinct_id")) { - yield* execute( - `ALTER TABLE ${CLICKHOUSE_EVENTS_TABLE} ADD COLUMN IF NOT EXISTS previous_distinct_id Nullable(String) AFTER distinct_id`, - ); - } - - const personColumns = yield* getColumnNames(CLICKHOUSE_PERSONS_TABLE); - yield* addColumnFromLegacy({ - columns: personColumns, - fallbackType: "String", - legacyExpression: legacyColumnExpression(personColumns, "customer_id"), - name: "person_id", - table: CLICKHOUSE_PERSONS_TABLE, - }); - yield* addColumnFromLegacy({ - columns: personColumns, - fallbackType: "Nullable(String)", - legacyExpression: legacyColumnExpression( - personColumns, - "app_user_id", - "nullIf(app_user_id, '')", - ), - name: "primary_distinct_id", - table: CLICKHOUSE_PERSONS_TABLE, - }); - yield* addColumnFromLegacy({ - columns: personColumns, - fallbackType: "String", - legacyExpression: legacyColumnExpression(personColumns, "properties"), - name: "traits", - table: CLICKHOUSE_PERSONS_TABLE, - }); - yield* addColumnFromLegacy({ - columns: personColumns, - fallbackType: "Nullable(String)", - legacyExpression: legacyColumnExpression(personColumns, "parent_customer_id"), - name: "merged_into_person_id", - table: CLICKHOUSE_PERSONS_TABLE, - }); - - const pendingOverrideColumns = yield* getColumnNames( - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - ); - yield* addColumnFromLegacy({ - columns: pendingOverrideColumns, - fallbackType: "String", - legacyExpression: legacyColumnExpression(pendingOverrideColumns, "customer_id"), - name: "person_id", - table: CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - }); -}); diff --git a/packages/clickhouse-db/src/migrations/0006_repair_pending_overrides_person_id.test.ts b/packages/clickhouse-db/src/migrations/0006_repair_pending_overrides_person_id.test.ts deleted file mode 100644 index 7e0dec427..000000000 --- a/packages/clickhouse-db/src/migrations/0006_repair_pending_overrides_person_id.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { analyticsEventsMigrations } from "../analytics/migration.ts"; -import { buildAddPendingOverridePersonIdStatement } from "./0006_repair_pending_overrides_person_id.ts"; - -describe("migration0006", () => { - it("adds a person_id compatibility column from legacy customer_id", () => { - expect(buildAddPendingOverridePersonIdStatement("person_identity_pending_overrides_v2")).toBe( - "ALTER TABLE person_identity_pending_overrides_v2 ADD COLUMN IF NOT EXISTS person_id String DEFAULT customer_id AFTER customer_id", - ); - }); - - it("is registered after the earlier alignment migration", () => { - expect(analyticsEventsMigrations.migrations.map(([id, name]) => [id, name])).toContainEqual([ - 6, - "0006_repair_pending_overrides_person_id", - ]); - }); -}); diff --git a/packages/clickhouse-db/src/migrations/0006_repair_pending_overrides_person_id.ts b/packages/clickhouse-db/src/migrations/0006_repair_pending_overrides_person_id.ts deleted file mode 100644 index 5e7a3b958..000000000 --- a/packages/clickhouse-db/src/migrations/0006_repair_pending_overrides_person_id.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE } from "../analytics/schema.ts"; -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -interface DescribeColumnRow { - name: string; -} - -export const buildAddPendingOverridePersonIdStatement = (table: string): string => - `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS person_id String DEFAULT customer_id AFTER customer_id`; - -const getColumnNames = (table: string) => - Effect.gen(function* getColumnNamesEffect() { - const sql = yield* SqlClient.SqlClient; - const rows = yield* sql` - DESCRIBE TABLE ${sql.literal(table)} - `; - - return new Set(rows.map((row) => row.name)); - }); - -const execute = (statement: string) => - Effect.gen(function* executeEffect() { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - yield* ch.asCommand(sql`${sql.literal(statement)}`); - }); - -export default Effect.gen(function* repairPendingOverridesPersonId() { - const columns = yield* getColumnNames(CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE); - - if (columns.has("person_id") || !columns.has("customer_id")) { - return; - } - - yield* execute( - buildAddPendingOverridePersonIdStatement(CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE), - ); -}); diff --git a/packages/clickhouse-db/src/migrations/0007_add_organization_id_to_identity_tables.ts b/packages/clickhouse-db/src/migrations/0007_add_organization_id_to_identity_tables.ts deleted file mode 100644 index 58fa1eeaa..000000000 --- a/packages/clickhouse-db/src/migrations/0007_add_organization_id_to_identity_tables.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { constant } from "@voidhash/lib/lang"; - -import { - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSONS_TABLE, -} from "../analytics/schema.ts"; -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -/** - * Person/identity tables that are keyed by `project_id` and gain an - * `organization_id` so the readonly analytics user's per-tenant row policies - * can isolate them (events_v2 already carries `organization_id`). - */ -const IDENTITY_TABLES = constant([ - CLICKHOUSE_PERSONS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, -]); - -/** Transient `project_id -> organization_id` lookup used only during backfill. */ -const ORG_BACKFILL_JOIN = "person_org_backfill_join_0007"; - -const execute = (statement: string) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - yield* ch.asCommand(sql`${sql.literal(statement)}`); - }); - -/** - * Adds `organization_id` to the four person/identity tables and backfills it - * from the authoritative `project_id -> organization_id` mapping carried by - * `events_v2`. New rows are stamped by the writer (it resolves the org from - * MySQL); this migration covers rows that predate the writer change. - * - * Idempotent: the `WHERE organization_id = ''` guard means a re-run after a - * crash only touches still-unset rows, and the `ADD COLUMN IF NOT EXISTS` / - * `DROP TABLE IF EXISTS` statements converge. Mutations run with - * `mutations_sync = 1` so the migration waits for each rewrite to finish. - * - * Known limitation: a project with person/identity rows but zero `events_v2` - * rows has no entry in the join and keeps `organization_id = ''` (invisible to - * the readonly user) until a new event for it lands. - */ -export default Effect.gen(function* () { - for (const table of IDENTITY_TABLES) { - yield* execute( - `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS organization_id String DEFAULT '' AFTER project_id`, - ); - } - - yield* execute(`DROP TABLE IF EXISTS ${ORG_BACKFILL_JOIN}`); - yield* execute( - `CREATE TABLE ${ORG_BACKFILL_JOIN} - (project_id String, organization_id String) - ENGINE = Join(ANY, LEFT, project_id)`, - ); - yield* execute( - `INSERT INTO ${ORG_BACKFILL_JOIN} - SELECT project_id, any(source_organization_id) AS organization_id - FROM ( - SELECT project_id, organization_id AS source_organization_id - FROM ${CLICKHOUSE_EVENTS_TABLE} - WHERE organization_id != '' - ) - GROUP BY project_id`, - ); - - for (const table of IDENTITY_TABLES) { - yield* execute( - `ALTER TABLE ${table} - UPDATE organization_id = joinGet('${ORG_BACKFILL_JOIN}', 'organization_id', project_id) - WHERE organization_id = '' - AND joinGet('${ORG_BACKFILL_JOIN}', 'organization_id', project_id) != '' - SETTINGS mutations_sync = 1`, - ); - } - - yield* execute(`DROP TABLE IF EXISTS ${ORG_BACKFILL_JOIN}`); -}); diff --git a/packages/clickhouse-db/src/migrations/0008_repair_identity_organization_id.test.ts b/packages/clickhouse-db/src/migrations/0008_repair_identity_organization_id.test.ts deleted file mode 100644 index 34481da76..000000000 --- a/packages/clickhouse-db/src/migrations/0008_repair_identity_organization_id.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { analyticsEventsMigrations } from "../analytics/migration.ts"; -import { - buildAddOrganizationIdStatement, - buildBackfillOrganizationIdStatement, - buildCreateOrgBackfillJoinStatement, - buildPopulateOrgBackfillJoinStatement, -} from "./0008_repair_identity_organization_id.ts"; - -describe("migration0008", () => { - it("adds the organization_id compatibility column", () => { - expect(buildAddOrganizationIdStatement("person_identity_pending_overrides_v2")).toBe( - "ALTER TABLE person_identity_pending_overrides_v2 ADD COLUMN IF NOT EXISTS organization_id String DEFAULT '' AFTER project_id", - ); - }); - - it("builds the transient backfill join", () => { - expect(buildCreateOrgBackfillJoinStatement("person_org_backfill_join_0008")).toContain( - "ENGINE = Join(ANY, LEFT, project_id)", - ); - expect(buildPopulateOrgBackfillJoinStatement("person_org_backfill_join_0008")).toContain( - "FROM events_v2", - ); - expect( - buildBackfillOrganizationIdStatement( - "person_identity_pending_overrides_v2", - "person_org_backfill_join_0008", - ), - ).toContain("SETTINGS mutations_sync = 1"); - }); - - it("is registered after the original organization-id migration", () => { - expect(analyticsEventsMigrations.migrations.map(([id, name]) => [id, name])).toContainEqual([ - 8, - "0008_repair_identity_organization_id", - ]); - }); -}); diff --git a/packages/clickhouse-db/src/migrations/0008_repair_identity_organization_id.ts b/packages/clickhouse-db/src/migrations/0008_repair_identity_organization_id.ts deleted file mode 100644 index a4ccd9d83..000000000 --- a/packages/clickhouse-db/src/migrations/0008_repair_identity_organization_id.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Effect } from "effect"; -import { SqlClient } from "effect/unstable/sql"; - -import { constant } from "@voidhash/lib/lang"; - -import { - CLICKHOUSE_EVENTS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSONS_TABLE, -} from "../analytics/schema.ts"; -import { ClickhouseWebClient } from "../clickhouse-client-web/index.ts"; - -const IDENTITY_TABLES = constant([ - CLICKHOUSE_PERSONS_TABLE, - CLICKHOUSE_PERSON_IDENTITY_TABLE, - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_TABLE, - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_TABLE, -]); - -const ORG_BACKFILL_JOIN = "person_org_backfill_join_0008"; - -/** Build the idempotent `organization_id` column repair for an identity table. */ -export const buildAddOrganizationIdStatement = (table: string): string => - `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS organization_id String DEFAULT '' AFTER project_id`; - -/** Build the transient project-to-organization join table used by the backfill. */ -export const buildCreateOrgBackfillJoinStatement = (joinTable: string): string => - `CREATE TABLE ${joinTable} - (project_id String, organization_id String) - ENGINE = Join(ANY, LEFT, project_id)`; - -/** Build the statement that loads the transient organization lookup from events. */ -export const buildPopulateOrgBackfillJoinStatement = (joinTable: string): string => - `INSERT INTO ${joinTable} - SELECT project_id, any(source_organization_id) AS organization_id - FROM ( - SELECT project_id, organization_id AS source_organization_id - FROM ${CLICKHOUSE_EVENTS_TABLE} - WHERE organization_id != '' - ) - GROUP BY project_id`; - -/** Build the mutation that fills missing organization ids for one identity table. */ -export const buildBackfillOrganizationIdStatement = (table: string, joinTable: string): string => - `ALTER TABLE ${table} - UPDATE organization_id = joinGet('${joinTable}', 'organization_id', project_id) - WHERE organization_id = '' - AND joinGet('${joinTable}', 'organization_id', project_id) != '' - SETTINGS mutations_sync = 1`; - -const execute = (statement: string) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const sql = yield* SqlClient.SqlClient; - yield* ch.asCommand(sql`${sql.literal(statement)}`); - }); - -export default Effect.gen(function* repairIdentityOrganizationId() { - for (const table of IDENTITY_TABLES) { - yield* execute(buildAddOrganizationIdStatement(table)); - } - - yield* execute(`DROP TABLE IF EXISTS ${ORG_BACKFILL_JOIN}`); - yield* execute(buildCreateOrgBackfillJoinStatement(ORG_BACKFILL_JOIN)); - yield* execute(buildPopulateOrgBackfillJoinStatement(ORG_BACKFILL_JOIN)); - - for (const table of IDENTITY_TABLES) { - yield* execute(buildBackfillOrganizationIdStatement(table, ORG_BACKFILL_JOIN)); - } - - yield* execute(`DROP TABLE IF EXISTS ${ORG_BACKFILL_JOIN}`); -}); diff --git a/packages/clickhouse-db/sst-env.d.ts b/packages/clickhouse-db/sst-env.d.ts deleted file mode 100644 index a73ac12f6..000000000 --- a/packages/clickhouse-db/sst-env.d.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* This file is auto-generated by SST. Do not edit. */ -/* tslint:disable */ -/* eslint-disable */ -/* deno-fmt-ignore-file */ - -/// - -import "sst"; -export {}; diff --git a/packages/clickhouse-db/tsconfig.json b/packages/clickhouse-db/tsconfig.json deleted file mode 100644 index 44cfab7dd..000000000 --- a/packages/clickhouse-db/tsconfig.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "compilerOptions": { "rootDir": "./src" }, - "extends": "@voidhash/tsconfig/internal-package-typescript-6.json", - "include": ["src"], - "exclude": ["**/node_modules/**"] -} diff --git a/packages/clickhouse-db/vitest.mts b/packages/clickhouse-db/vitest.mts deleted file mode 100644 index c37b0a43f..000000000 --- a/packages/clickhouse-db/vitest.mts +++ /dev/null @@ -1,11 +0,0 @@ -import { defineConfig } from "vite-plus"; - -export default defineConfig({ - test: { - environment: "node", - include: ["./**/*.test.ts"], - exclude: ["./**/*.integration.test.ts", "./node_modules/**", "./dist/**"], - reporters: ["verbose"], - passWithNoTests: true, - }, -}); diff --git a/packages/core/package.json b/packages/core/package.json index 286ef069e..572cb00d7 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -34,6 +34,7 @@ "./utils/crypto/account-token": "./src/utils/crypto/account-token.ts", "./utils/crypto/standalone-auth-token": "./src/utils/crypto/standalone-auth-token.ts", "./utils/generate-id": "./src/utils/generate-id.ts", + "./utils/permissions": "./src/utils/permissions.ts", "./services/personIdentity/IdentityMutationService": "./src/services/personIdentity/IdentityMutationService.ts", "./services/paymentProviders/appStore/payment-provider": "./src/services/paymentProviders/appStore/payment-provider.ts", "./services/paymentProviders/appStore/payment-provider-service-queries": "./src/services/paymentProviders/appStore/payment-provider-service-queries.ts", @@ -46,20 +47,12 @@ "./services/paymentProviders/googlePlay/webhook-handler-service": "./src/services/paymentProviders/googlePlay/webhook-handler-service.ts", "./services/paymentProviders/googlePlay/sdk-context": "./src/services/paymentProviders/googlePlay/sdk-context.ts", "./domain/analytics/Analytics": "./src/domain/analytics/Analytics.ts", - "./domain/analyticsIngest/AnalyticsIngest": "./src/domain/analyticsIngest/AnalyticsIngest.ts", + "./domain/analytics/AnalyticsEvent": "./src/domain/analytics/AnalyticsEvent.ts", "./services": "./src/services/index.ts", "./services/analytics/AnalyticsService": "./src/services/analytics/AnalyticsService.ts", - "./services/analytics/CustomAnalyticsService": "./src/services/analytics/CustomAnalyticsService.ts", + "./services/analytics/AnalyticsEventStore": "./src/services/analytics/AnalyticsEventStore.ts", "./services/analyticsIngest/AnalyticsDispatchService": "./src/services/analyticsIngest/AnalyticsDispatchService.ts", - "./services/analyticsIngest/AnalyticsIngestDlqService": "./src/services/analyticsIngest/AnalyticsIngestDlqService.ts", - "./services/analyticsIngest/AnalyticsJanitorService": "./src/services/analyticsIngest/AnalyticsJanitorService.ts", - "./services/analyticsIngest/AnalyticsWriterService": "./src/services/analyticsIngest/AnalyticsWriterService.ts", - "./services/analyticsIngest/CaptureIngress": "./src/services/analyticsIngest/CaptureIngress.ts", - "./services/analyticsIngest/DlqProducer": "./src/services/analyticsIngest/DlqProducer.ts", "./services/analyticsIngest/EventCaptureService": "./src/services/analyticsIngest/EventCaptureService.ts", - "./services/analyticsIngest/EventProcessorService": "./src/services/analyticsIngest/EventProcessorService.ts", - "./services/analyticsIngest/PolicyCounterStore": "./src/services/analyticsIngest/PolicyCounterStore.ts", - "./services/analyticsIngest/ProcessorOutputs": "./src/services/analyticsIngest/ProcessorOutputs.ts", "./services/apiKeys/ApiKeyService": "./src/services/apiKeys/ApiKeyService.ts", "./services/auditLog/AuditLogPort": "./src/services/auditLog/AuditLogPort.ts", "./services/auth/AuthTokenVerifier": "./src/services/auth/AuthTokenVerifier.ts", @@ -124,7 +117,6 @@ "./services/storage/PublicFileStore": "./src/services/storage/PublicFileStore.ts", "./eventBus/CoreEventBus": "./src/eventBus/CoreEventBus.ts", "./services/paywallLocations/PaywallAssetConfig": "./src/services/paywallLocations/PaywallAssetConfig.ts", - "./services/infrastructure/Clickhouse": "./src/services/infrastructure/Clickhouse.ts", "./services/infrastructure/QueueProducer": "./src/services/infrastructure/QueueProducer.ts", "./services/notifications/PushDeliveryDispatch": "./src/services/notifications/PushDeliveryDispatch.ts", "./services/notifications/PushDeliveryService": "./src/services/notifications/PushDeliveryService.ts", @@ -149,11 +141,9 @@ }, "dependencies": { "@distilled.cloud/stripe": "0.30.3", - "@effect/sql-clickhouse": "catalog:", "@paralleldrive/cuid2": "^2.2.2", "@voidhash/api-contracts": "workspace:*", "@voidhash/app-store-server-sdk": "workspace:*", - "@voidhash/clickhouse-db": "workspace:*", "@voidhash/db": "workspace:*", "@voidhash/google-play-server-sdk": "workspace:*", "@voidhash/lib": "workspace:*", diff --git a/packages/core/src/domain/analytics/Analytics.ts b/packages/core/src/domain/analytics/Analytics.ts index 8a5f7a034..f90877108 100644 --- a/packages/core/src/domain/analytics/Analytics.ts +++ b/packages/core/src/domain/analytics/Analytics.ts @@ -436,9 +436,8 @@ export const ensureNoBreakdowns = ( // ============================================================================= /** - * Truncate sub-second precision. ClickHouse's `DateTime` column rejects - * fractional-second timestamps when bound as query parameters, and analytics - * buckets never care about sub-seconds, so we floor at the resolver boundary. + * Truncate sub-second precision because analytics buckets do not distinguish + * values within the same second. */ const truncateToSecond = (date: Date): Date => fromEpochMillis(Math.floor(date.getTime() / 1000) * 1000); diff --git a/packages/core/src/domain/analytics/AnalyticsEvent.ts b/packages/core/src/domain/analytics/AnalyticsEvent.ts new file mode 100644 index 000000000..89b2d594d --- /dev/null +++ b/packages/core/src/domain/analytics/AnalyticsEvent.ts @@ -0,0 +1,214 @@ +import type { CaptureEvent } from "@voidhash/api-contracts/event-capture"; +import { constant } from "@voidhash/lib/lang"; +import { DateTime, Schema } from "effect"; + +import { + sourceTopicForInternalAnalyticsEvent, + type InternalAnalyticsEvent, +} from "../internalAnalytics/InternalAnalyticsEvents.ts"; + +/** SDK events retained by the Community analytics implementation. */ +export const COMMUNITY_CAPTURE_EVENT_NAMES = constant([ + "$app_installed", + "$app_updated", + "$app_opened", + "$app_backgrounded", + "$app_became_active", + "$sign_out", +]); + +export type CommunityCaptureEventName = (typeof COMMUNITY_CAPTURE_EVENT_NAMES)[number]; + +const communityCaptureEventNames: ReadonlySet = new Set(COMMUNITY_CAPTURE_EVENT_NAMES); + +/** Whether a public SDK event is supported by Community analytics. */ +export const isCommunityCaptureEventName = ( + eventName: string, +): eventName is CommunityCaptureEventName => communityCaptureEventNames.has(eventName); + +export const AnalyticsEventSource = Schema.Literals(["sdk", "revenue", "internal"]); +export type AnalyticsEventSource = typeof AnalyticsEventSource.Type; + +/** + * Storage-neutral event shared by the PostgreSQL and hosted analytics + * implementations. The fields intentionally track the hosted processed-event + * record so a Community export can be imported without semantic remapping. + */ +export const AnalyticsEventV1 = Schema.Struct({ + schemaVersion: Schema.Literal(1), + eventId: Schema.String, + captureId: Schema.String, + eventName: Schema.String, + eventTimestamp: Schema.Date, + processedAt: Schema.Date, + organizationId: Schema.String, + projectId: Schema.String, + distinctId: Schema.String, + previousDistinctId: Schema.NullOr(Schema.String), + personId: Schema.NullOr(Schema.String), + identityMode: Schema.Literals(["full", "personless"]), + properties: Schema.Record(Schema.String, Schema.Unknown), + context: Schema.Record(Schema.String, Schema.Unknown), + sessionId: Schema.NullOr(Schema.String), + token: Schema.String, + requestId: Schema.String, + requestPath: Schema.NullOr(Schema.String), + source: AnalyticsEventSource, + sourceTopic: Schema.String, +}); + +export type AnalyticsEventV1 = typeof AnalyticsEventV1.Type; + +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); +const decodeJsonRecord = Schema.decodeUnknownSync( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + +const normalizeJsonRecord = (value: unknown) => decodeJsonRecord(encodeJson(value ?? {})); + +const storedProperties = (value: Readonly>) => { + const inner = value.properties; + if (typeof inner !== "object" || inner === null || Array.isArray(inner)) return value; + return normalizeJsonRecord(inner); +}; + +const identityModeForPerson = (personId: string | null): "full" | "personless" => { + if (personId) return "full"; + return "personless"; +}; + +const sourceForInternalEvent = (eventName: string): AnalyticsEventSource => { + if (eventName === "$experiment.exposed") return "internal"; + return "revenue"; +}; + +const previousDistinctIdFrom = (properties: Readonly>): string | null => { + if (typeof properties.$previous_distinct_id === "string") { + return properties.$previous_distinct_id; + } + return null; +}; + +const sourceForHostedTopic = (sourceTopic: string): AnalyticsEventSource => { + if (sourceTopic.startsWith("revenue.")) return "revenue"; + return "sdk"; +}; + +/** Maps an allow-listed SDK capture onto the portable event contract. */ +export const analyticsEventFromCapture = (input: { + readonly event: typeof CaptureEvent.Type; + readonly organizationId: string; + readonly projectId: string; + readonly receivedAt: Date; + readonly requestId: string; + readonly requestPath?: string; + readonly sentAt: Date; + readonly token: string; +}): AnalyticsEventV1 => { + const eventTimestamp = input.event.timestamp ?? input.sentAt ?? input.receivedAt; + return { + schemaVersion: 1, + eventId: input.event.uuid, + captureId: `capture_${input.event.uuid}`, + eventName: input.event.event, + eventTimestamp, + processedAt: input.receivedAt, + organizationId: input.organizationId, + projectId: input.projectId, + distinctId: input.event.distinct_id, + previousDistinctId: null, + personId: null, + identityMode: "personless", + properties: normalizeJsonRecord(input.event.properties), + context: input.event.context, + sessionId: input.event.session_id ?? null, + token: input.token, + requestId: input.requestId, + requestPath: input.requestPath ?? null, + source: "sdk", + sourceTopic: "community.capture.v1", + }; +}; + +/** Maps a server-trusted event onto the same portable event contract. */ +export const analyticsEventFromInternal = ( + event: InternalAnalyticsEvent, + processedAt: Date = DateTime.toDateUtc(DateTime.nowUnsafe()), +): AnalyticsEventV1 => ({ + schemaVersion: 1, + eventId: event.eventId, + captureId: `internal_${event.eventId}`, + eventName: event.eventName, + eventTimestamp: event.occurredAt, + processedAt, + organizationId: event.organizationId, + projectId: event.projectId, + distinctId: event.distinctId, + previousDistinctId: null, + personId: event.personId, + identityMode: identityModeForPerson(event.personId), + properties: normalizeJsonRecord(event.properties), + context: normalizeJsonRecord(event.context), + sessionId: null, + token: event.token, + requestId: `internal_${event.eventId}`, + requestPath: "/internal/analytics", + source: sourceForInternalEvent(event.eventName), + sourceTopic: sourceTopicForInternalAnalyticsEvent(event), +}); + +/** + * Structural hosted processed-event shape accepted by the compatibility + * mapper. Kept independent of the queue schema so shared tests do not depend + * on a particular transport implementation. + */ +export interface HostedProcessedAnalyticsEvent { + readonly captureId: string; + readonly context: Readonly>; + readonly distinctId: string; + readonly event: string; + readonly eventTimestamp: string; + readonly identity: { + readonly distinctId: string; + readonly mode: "full" | "personless"; + readonly personId?: string; + }; + readonly organizationId: string; + readonly processedAt: string; + readonly processedEventId: string; + readonly projectId: string; + readonly properties: Readonly>; + readonly request: { readonly path?: string; readonly requestId: string }; + readonly routing: { readonly sourceTopic: string }; + readonly sessionId?: string; + readonly token: string; +} + +/** Maps the hosted processed-event envelope onto the portable contract. */ +export const analyticsEventFromHostedProcessed = ( + event: HostedProcessedAnalyticsEvent, +): AnalyticsEventV1 => { + const properties = storedProperties(event.properties); + return { + schemaVersion: 1, + eventId: event.processedEventId, + captureId: event.captureId, + eventName: event.event, + eventTimestamp: DateTime.toDateUtc(DateTime.makeUnsafe(event.eventTimestamp)), + processedAt: DateTime.toDateUtc(DateTime.makeUnsafe(event.processedAt)), + organizationId: event.organizationId, + projectId: event.projectId, + distinctId: event.identity.distinctId, + previousDistinctId: previousDistinctIdFrom(properties), + personId: event.identity.personId ?? null, + identityMode: event.identity.mode, + properties, + context: event.context, + sessionId: event.sessionId ?? null, + token: event.token, + requestId: event.request.requestId, + requestPath: event.request.path ?? null, + source: sourceForHostedTopic(event.routing.sourceTopic), + sourceTopic: event.routing.sourceTopic, + }; +}; diff --git a/packages/core/src/domain/analyticsIngest/AnalyticsIngest.ts b/packages/core/src/domain/analyticsIngest/AnalyticsIngest.ts deleted file mode 100644 index e1f9d08d7..000000000 --- a/packages/core/src/domain/analyticsIngest/AnalyticsIngest.ts +++ /dev/null @@ -1,817 +0,0 @@ -/** - * Analytics-ingest domain. Consolidates the wire-stable event schemas the - * capture/processor/writer/janitor pipeline exchanges, plus the pure helpers - * (envelope builders, DLQ builders, person-trait parsers, validation rules) - * that the services rely on. - * - * Wire-stable schemas (consumers across processes must keep these stable): - * - {@link CapturedEventV1} — capture → processor envelope. - * - {@link ProcessedEventV2} — processor → writer (analytics events). - * - {@link ProcessorPersonEventV1} — processor → writer (person snapshot). - * - {@link ProcessorPersonIdentityEventV1} — processor → writer (identity map). - * - {@link AnalyticsWriterMessage} — the writer's tagged input union. - * - {@link EventProcessorDlqV1} — processor → DLQ. - */ -import { createId } from "@paralleldrive/cuid2"; -import { DateTime, Option, Schema } from "effect"; -import { - sourceTopicForInternalAnalyticsEvent, - type InternalAnalyticsEvent, -} from "../internalAnalytics/InternalAnalyticsEvents.ts"; - -/** `JSON.stringify` equivalent for the JSON text columns / round-trips below. */ -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); - -// ============================================================================= -// Captured event (capture → processor) -// ============================================================================= - -export type RouteClass = "main" | "dlq" | "overflow" | "historical" | "custom"; - -export const CapturedEventRequest = Schema.Struct({ - path: Schema.optional(Schema.String), - requestId: Schema.String, - userAgent: Schema.optional(Schema.String), - clientIp: Schema.optional(Schema.String), - isInternal: Schema.optional(Schema.Boolean), -}); -export type CapturedEventRequest = typeof CapturedEventRequest.Type; - -export const CapturedEventRouting = Schema.Struct({ - routeClass: Schema.Literals(["main", "dlq", "overflow", "historical", "custom"]), - targetTopic: Schema.String, - isHistorical: Schema.Boolean, - skipEnrichment: Schema.Boolean, -}); -export type CapturedEventRouting = typeof CapturedEventRouting.Type; - -type EventPropertiesFieldPrimitive = string | number | boolean | null; -type EventPropertiesField = - | EventPropertiesFieldPrimitive - | ReadonlyArray - | { readonly [key: string]: EventPropertiesField }; - -const EventPropertiesFieldSchema: Schema.Codec = Schema.Union([ - Schema.String, - Schema.Finite, - Schema.Boolean, - Schema.Null, - Schema.Array( - Schema.suspend((): Schema.Codec => EventPropertiesFieldSchema), - ), - Schema.Record( - Schema.String, - Schema.suspend((): Schema.Codec => EventPropertiesFieldSchema), - ), -]); -const EventPropertiesSchema = Schema.Record(Schema.String, EventPropertiesFieldSchema); - -type EventContextFieldPrimitive = string | number | boolean | null; -type EventContextField = - | EventContextFieldPrimitive - | ReadonlyArray - | { readonly [key: string]: EventContextField }; - -const EventContextFieldSchema: Schema.Codec = Schema.Union([ - Schema.String, - Schema.Finite, - Schema.Boolean, - Schema.Null, - Schema.Array(Schema.suspend((): Schema.Codec => EventContextFieldSchema)), - Schema.Record( - Schema.String, - Schema.suspend((): Schema.Codec => EventContextFieldSchema), - ), -]); -const EventContextSchema = Schema.Record(Schema.String, EventContextFieldSchema); - -/** - * The identity the capture layer asserts for an event, as a tagged claim the - * processor honours: - * - `Anonymous` — SDK pre-identify; the processor resolves / creates identity. - * - `Stitch` — SDK `$identify`; the processor stitches `previousDistinctId`. - * - `Resolved` — a server-trusted caller (revenue) already knows the person; - * the processor passes the `(distinctId, personId)` through and - * writes NO `persons_v1` / `person_identity_v1` rows. - * - * Optional on {@link CapturedEventV1} so in-flight messages (and the SDK path, - * until it stamps a claim) still decode; the processor only special-cases - * `Resolved`, and only when the event is server-trusted. - */ -export const CapturedIdentityClaim = Schema.Union([ - Schema.Struct({ _tag: Schema.Literal("Anonymous"), distinctId: Schema.String }), - Schema.Struct({ - _tag: Schema.Literal("Stitch"), - distinctId: Schema.String, - previousDistinctId: Schema.String, - }), - Schema.Struct({ - _tag: Schema.Literal("Resolved"), - distinctId: Schema.String, - personId: Schema.String, - }), -]); -export type CapturedIdentityClaim = typeof CapturedIdentityClaim.Type; - -/** - * Origin / trust of a captured event. The ingest queue is internal-only and - * this marker is stamped SERVER-SIDE at the dispatch boundary (never threaded - * from request input): `untrusted-sdk` for the public capture path, - * `trusted-revenue` for server-emitted revenue events. - */ -export const TrustClass = Schema.Literals(["untrusted-sdk", "trusted-revenue", "trusted-internal"]); -export type TrustClass = typeof TrustClass.Type; - -export const CapturedEventV1 = Schema.Struct({ - schemaVersion: Schema.Literal(1), - captureId: Schema.String, - clientEventId: Schema.optional(Schema.String), - sessionId: Schema.optional(Schema.String), - token: Schema.String, - organizationId: Schema.String, - projectId: Schema.String, - event: Schema.String, - distinctId: Schema.String, - eventTimestamp: Schema.String, - receivedAt: Schema.String, - sentAt: Schema.optional(Schema.String), - properties: EventPropertiesSchema, - context: EventContextSchema, - rawPayload: Schema.Record(Schema.String, Schema.Unknown), - request: CapturedEventRequest, - routing: CapturedEventRouting, - identityClaim: Schema.optional(CapturedIdentityClaim), - trustClass: Schema.optional(TrustClass), -}); -export type CapturedEventV1Type = typeof CapturedEventV1.Type; - -// ============================================================================= -// Processed event (processor → writer) -// ============================================================================= - -export const ProcessorLane = Schema.Literals(["historical", "main", "overflow"]); -export type ProcessorLane = typeof ProcessorLane.Type; - -/** - * Wire schema for one analytics-ingest queue message: an accepted capture - * envelope plus the supported lane it routes to. Unsupported routes never reach - * the queue (the producer records them in the ingest DLQ), so `lane` is a - * {@link ProcessorLane}. - */ -export const AnalyticsIngestQueueMessage = Schema.Struct({ - envelope: CapturedEventV1, - lane: ProcessorLane, -}); -export type AnalyticsIngestQueueMessageType = typeof AnalyticsIngestQueueMessage.Type; - -export const ProcessedEventIdentity = Schema.Struct({ - personId: Schema.optional(Schema.String), - distinctId: Schema.String, - mode: Schema.Literals(["full", "personless"]), -}); -export type ProcessedEventIdentity = typeof ProcessedEventIdentity.Type; - -export const ProcessedEventRouting = Schema.Struct({ - lane: ProcessorLane, - skipEnrichment: Schema.Boolean, - sourceOffset: Schema.String, - sourcePartition: Schema.Number, - sourceTopic: Schema.String, -}); -export type ProcessedEventRouting = typeof ProcessedEventRouting.Type; - -export const ProcessedEventV2 = Schema.Struct({ - captureId: Schema.String, - context: Schema.Record(Schema.String, Schema.Unknown), - distinctId: Schema.String, - event: Schema.String, - eventTimestamp: Schema.String, - groups: Schema.Array(Schema.Never), - identity: ProcessedEventIdentity, - organizationId: Schema.String, - processedAt: Schema.String, - processedEventId: Schema.String, - projectId: Schema.String, - properties: Schema.Record(Schema.String, Schema.Unknown), - request: CapturedEventRequest, - routing: ProcessedEventRouting, - schemaVersion: Schema.Literal(2), - sessionId: Schema.optional(Schema.String), - token: Schema.String, -}); -export type ProcessedEventV2Type = typeof ProcessedEventV2.Type; - -// ============================================================================= -// Processor outputs (person snapshot / identity event) -// ============================================================================= - -export const ProcessorPersonEventV1 = Schema.Struct({ - changedAt: Schema.String, - personId: Schema.String, - email: Schema.optional(Schema.String), - isArchived: Schema.Boolean, - mergedIntoPersonId: Schema.optional(Schema.String), - name: Schema.optional(Schema.String), - primaryDistinctId: Schema.optional(Schema.String), - projectId: Schema.String, - schemaVersion: Schema.Literal(1), - traits: Schema.Record(Schema.String, Schema.Unknown), - version: Schema.Number, -}); -export type ProcessorPersonEventV1Type = typeof ProcessorPersonEventV1.Type; - -export const ProcessorPersonIdentityEventV1 = Schema.Struct({ - changedAt: Schema.String, - personId: Schema.String, - distinctId: Schema.String, - isDeleted: Schema.Boolean, - previousDistinctId: Schema.optional(Schema.String), - projectId: Schema.String, - schemaVersion: Schema.Literal(1), - version: Schema.Number, -}); -export type ProcessorPersonIdentityEventV1Type = typeof ProcessorPersonIdentityEventV1.Type; - -// ============================================================================= -// Capture route + project policy -// ============================================================================= - -export interface CaptureProjectPolicy { - readonly customTopic?: string; - readonly eventsPerDay?: number; - readonly forceRoute?: RouteClass; - readonly ingestEnabled: boolean; - readonly projectId: string; - readonly requestsPerMinute?: number; - readonly skipEnrichment: boolean; -} - -export interface RouteDecision { - readonly isHistorical: boolean; - readonly routeClass: RouteClass; - readonly skipEnrichment: boolean; - readonly targetTopic: string; -} - -export const defaultCaptureProjectPolicy = (projectId: string): CaptureProjectPolicy => ({ - ingestEnabled: true, - projectId, - skipEnrichment: false, -}); - -// ============================================================================= -// Transport record + DLQ event -// ============================================================================= - -export interface CapturedTransportRecord { - readonly capturedEvent: CapturedEventV1Type; - readonly headers: Readonly>; - readonly lane: ProcessorLane; - readonly rawKey?: string; - readonly rawValue: string; - readonly sourceOffset: string; - readonly sourcePartition: number; - readonly sourceTopic: string; -} - -export interface EventProcessorDlqV1 { - readonly captureId?: string; - readonly distinctId?: string; - readonly failedAt: string; - readonly failureClass: - | "captured_event_invalid" - | "policy_rejected" - | "project_not_found" - | "reserved_event_name" - | "schema_rejected" - | "transport_invalid" - | "unsupported_lane"; - readonly failureId: string; - readonly failureMessage: string; - readonly headers: Record; - readonly lane: ProcessorLane | "unknown"; - readonly projectId?: string; - readonly rawKey?: string; - readonly rawValue?: string; - readonly schemaVersion: 1; - readonly sourceOffset: string; - readonly sourcePartition: number; - readonly sourceTopic: string; - readonly token?: string; -} - -export const buildDlqEvent = ({ - captureId, - distinctId, - failureClass, - failureMessage, - headers, - lane, - projectId, - rawKey, - rawValue, - sourceOffset, - sourcePartition, - sourceTopic, - token, -}: Omit): EventProcessorDlqV1 => { - // Absent optionals are omitted entirely (never present-but-undefined), so the - // DLQ record stays minimal on the wire. - const optional: { - -readonly [K in "captureId" | "distinctId" | "projectId" | "rawKey" | "rawValue" | "token"]?: - | EventProcessorDlqV1[K] - | undefined; - } = {}; - if (captureId) optional.captureId = captureId; - if (distinctId) optional.distinctId = distinctId; - if (projectId) optional.projectId = projectId; - if (rawKey) optional.rawKey = rawKey; - if (rawValue) optional.rawValue = rawValue; - if (token) optional.token = token; - - return { - ...optional, - failedAt: DateTime.formatIso(DateTime.nowUnsafe()), - failureClass, - // oxlint-disable-next-line effect/noGlobals -- Effect v4's Crypto is a Context.Service with no platform-neutral layer in the `effect` barrel (Node/Browser/Bun only, none Workers-safe); buildDlqEvent is a synchronous pure builder, so requiring Crypto would force every DLQ call site into an Effect. - failureId: crypto.randomUUID(), - failureMessage, - headers, - lane, - schemaVersion: 1, - sourceOffset, - sourcePartition, - sourceTopic, - }; -}; - -// ============================================================================= -// Person-trait helpers (pure) -// ============================================================================= - -export interface PersonTraits { - readonly setOnce: Record; - readonly set: Record; -} - -const isPlainRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); - -export const parsePersonTraits = ( - properties: Record, -): - | { readonly ok: true; readonly value: PersonTraits } - | { readonly ok: false; readonly message: string } => { - const rawSet = properties.$set; - if (typeof rawSet !== "undefined" && !isPlainRecord(rawSet)) { - return { message: "$set must be an object", ok: false }; - } - - const rawSetOnce = properties.$set_once; - if (typeof rawSetOnce !== "undefined" && !isPlainRecord(rawSetOnce)) { - return { message: "$set_once must be an object", ok: false }; - } - - return { - ok: true, - value: { - set: rawSet ?? {}, - setOnce: rawSetOnce ?? {}, - }, - }; -}; - -export const extractInnerProperties = ( - wrappedProperties: Record, -): Record => { - const inner = wrappedProperties.properties; - if (isPlainRecord(inner)) { - return inner; - } - return wrappedProperties; -}; - -/** - * Epoch milliseconds for an ISO timestamp, or `NaN` when it cannot be parsed — - * mirroring `new Date(value).getTime()` for the callers that compare ages. - */ -const epochMillisOf = (value: string): number => { - const parsed = DateTime.make(value); - if (Option.isNone(parsed)) { - return Number.NaN; - } - return DateTime.toEpochMillis(parsed.value); -}; - -// ============================================================================= -// Processor project policy + validation rules -// ============================================================================= - -export interface ProcessorProjectPolicy { - readonly processorAllowHistorical: boolean; - readonly processorAllowOverflow: boolean; - readonly processorEnabled: boolean; - readonly processorHistoricalMinAgeHours: number; - readonly processorPersonProcessingEnabled: boolean; - readonly processorSchemaMode: string; -} - -export interface ResolvedProcessorProject { - readonly organizationId: string; - readonly policy: ProcessorProjectPolicy; - readonly projectId: string; -} - -export interface ProcessingEvent { - readonly capturedEvent: CapturedEventV1Type; - readonly headers: Readonly>; - readonly identityKey: string; - readonly lane: ProcessorLane; - readonly projectPolicy: ProcessorProjectPolicy; - readonly rawKey?: string; - readonly rawValue: string; - readonly sourceOffset: string; - readonly sourcePartition: number; - readonly sourceTopic: string; -} - -export const ANONYMOUS_DISTINCT_ID_PREFIX = "vh:anon:"; - -export const validateBuiltInProcessorRules = ({ - capturedEvent, - historicalMinAgeHours, - lane, - now, - sourceTopic, -}: { - readonly capturedEvent: CapturedEventV1Type; - readonly historicalMinAgeHours: number; - readonly lane: ProcessorLane; - readonly now: Date; - readonly sourceTopic: string; -}): string | undefined => { - if (capturedEvent.routing.targetTopic !== sourceTopic) { - return "captured event routing target does not match source topic"; - } - - if (lane === "historical") { - if (!capturedEvent.routing.isHistorical) { - return "historical topic requires isHistorical=true"; - } - const eventAgeMs = now.getTime() - epochMillisOf(capturedEvent.eventTimestamp); - const minimumAgeMs = historicalMinAgeHours * 60 * 60 * 1000; - if (eventAgeMs < minimumAgeMs) { - return "historical event is newer than the configured minimum age"; - } - } else if (capturedEvent.routing.isHistorical) { - return "non-historical lane received a historical captured event"; - } - - const innerProperties = extractInnerProperties(capturedEvent.properties); - - const traitsResult = parsePersonTraits(innerProperties); - if (!traitsResult.ok) return traitsResult.message; - - const rawProcessPersonProfile = capturedEvent.properties.$process_person_profile; - if ( - typeof rawProcessPersonProfile !== "undefined" && - typeof rawProcessPersonProfile !== "boolean" - ) { - return "$process_person_profile must be a boolean"; - } - - if (capturedEvent.event === "$identify") { - const rawPreviousDistinctId = innerProperties.$previous_distinct_id; - if (typeof rawPreviousDistinctId !== "string" || rawPreviousDistinctId.length === 0) { - return "$identify requires properties.$previous_distinct_id"; - } - if (capturedEvent.distinctId.startsWith(ANONYMOUS_DISTINCT_ID_PREFIX)) { - return "$identify target distinct id cannot use the anonymous prefix"; - } - } - - return undefined; -}; - -// ============================================================================= -// Analytics writer message + row builders -// ============================================================================= - -export const AnalyticsWriterMessage = Schema.Union([ - Schema.Struct({ - kind: Schema.Literal("processed"), - messageId: Schema.String, - value: ProcessedEventV2, - }), - Schema.Struct({ - kind: Schema.Literal("person"), - messageId: Schema.String, - value: ProcessorPersonEventV1, - }), - Schema.Struct({ - kind: Schema.Literal("person-distinct-id"), - messageId: Schema.String, - value: ProcessorPersonIdentityEventV1, - }), -]); - -const decodeEventProperties = Schema.decodeUnknownSync(Schema.fromJsonString(EventPropertiesSchema)); -const decodeEventContext = Schema.decodeUnknownSync(Schema.fromJsonString(EventContextSchema)); - -/** - * Revenue always knows the person (`Resolved`); experiment exposure may fire for - * an anonymous viewer (`personId` null → `Anonymous`, resolved on read). - */ -const capturedIdentityClaim = (event: InternalAnalyticsEvent): CapturedIdentityClaim => { - if (event.personId) { - return { _tag: "Resolved", distinctId: event.distinctId, personId: event.personId }; - } - return { _tag: "Anonymous", distinctId: event.distinctId }; -}; - -const capturedTrustClass = (event: InternalAnalyticsEvent): TrustClass => { - if (event.eventName === "$experiment.exposed") { - return "trusted-internal"; - } - return "trusted-revenue"; -}; - -/** - * Maps a server-trusted {@link InternalAnalyticsEvent} into a - * {@link CapturedEventV1} for the SHARED ingest queue — the revenue transport's - * entry into the same pipeline the SDK uses. - * - * - The deterministic `event.eventId` becomes `clientEventId`, so it flows - * through `buildProcessedEvent` (`clientEventId ?? captureId`) to - * `events_v2.event_id` unchanged — the dedup key. - * - A `Resolved` identity claim + the trusted source topic tell the processor - * to pass `(distinctId, personId)` through and emit NO person/identity rows - * (exactly today's revenue behaviour). - * - `skipEnrichment` is set so the processor never runs person enrichment. - * - `properties` is JSON round-tripped so Date values (e.g. `transferredAt`) - * become ISO strings — the wire `properties` schema permits only JSON - * primitives, and the result matches the stored JSON text form. - */ -export const makeCapturedEventFromInternalAnalyticsEvent = ( - event: InternalAnalyticsEvent, -): CapturedEventV1Type => { - const captureId = `internal_${event.eventId}`; - const targetTopic = sourceTopicForInternalAnalyticsEvent(event); - const properties = decodeEventProperties(encodeJson(event.properties)); - const context = decodeEventContext(encodeJson(event.context ?? {})); - const identityClaim = capturedIdentityClaim(event); - const trustClass = capturedTrustClass(event); - return { - schemaVersion: 1, - captureId, - clientEventId: event.eventId, - token: event.token, - organizationId: event.organizationId, - projectId: event.projectId, - event: event.eventName, - distinctId: event.distinctId, - eventTimestamp: event.occurredAt.toISOString(), - receivedAt: DateTime.formatIso(DateTime.nowUnsafe()), - properties, - context, - rawPayload: {}, - request: { - path: "/internal/analytics", - requestId: captureId, - }, - routing: { - routeClass: "main", - targetTopic, - isHistorical: false, - skipEnrichment: true, - }, - identityClaim, - trustClass, - }; -}; - -export type AnalyticsWriterMessageType = typeof AnalyticsWriterMessage.Type; - -export interface AnalyticsWriterPlan { - readonly personIdentityOverrideRows: ReadonlyArray>; - readonly personIdentityPendingOverrideRows: ReadonlyArray>; - readonly personIdentityRows: ReadonlyArray>; - readonly personRows: ReadonlyArray>; - readonly processedEventRows: ReadonlyArray>; -} - -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); - -const asRecord = (value: unknown): Record => { - if (isRecord(value)) { - return value; - } - return {}; -}; - -const toNullableString = (value: string | undefined): string | null => { - if (typeof value === "string" && value.trim().length > 0) { - return value; - } - return null; -}; - -export const toFlag = (value: boolean): 0 | 1 => { - if (value) { - return 1; - } - return 0; -}; - -export const toClickhouseTimestamp = (value: string): string => { - const parsed = DateTime.make(value); - if (Option.isNone(parsed)) { - // oxlint-disable-next-line effect/noThrowStatement, effect/noNewError -- synchronous ClickHouse row mapper used inside plain object literals (see the row builders below); it has no Effect channel, and an unparseable timestamp at this point is a defect. - throw new Error(`Invalid timestamp: ${value}`); - } - const date = DateTime.toDateUtc(parsed.value); - const pad = (part: number, length = 2) => String(part).padStart(length, "0"); - return [ - `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`, - `${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())}.${pad( - date.getUTCMilliseconds(), - 3, - )}`, - ].join(" "); -}; - -export const extractPreviousDistinctId = (event: ProcessedEventV2Type): string | null => { - const wrappedProperties = asRecord(event.properties); - const innerProperties = extractInnerProperties(wrappedProperties); - const previousDistinctId = innerProperties.$previous_distinct_id; - if (typeof previousDistinctId !== "string") { - return null; - } - return toNullableString(previousDistinctId); -}; - -export const toProcessedEventRow = (event: ProcessedEventV2Type): Record => ({ - capture_id: event.captureId, - context: encodeJson(event.context), - person_id: toNullableString(event.identity.personId), - distinct_id: event.identity.distinctId, - event_id: event.processedEventId, - event_name: event.event, - event_properties: encodeJson(event.properties), - event_ts: toClickhouseTimestamp(event.eventTimestamp), - identity_mode: event.identity.mode, - organization_id: event.organizationId, - previous_distinct_id: extractPreviousDistinctId(event), - processed_ts: toClickhouseTimestamp(event.processedAt), - project_id: event.projectId, - request_id: event.request.requestId, - request_path: event.request.path ?? "", - route_lane: event.routing.lane, - schema_version: event.schemaVersion, - skip_enrichment: toFlag(event.routing.skipEnrichment), - source_offset: event.routing.sourceOffset, - source_partition: event.routing.sourcePartition, - source_topic: event.routing.sourceTopic, - token: event.token, -}); - -export const toPersonRow = ( - event: ProcessorPersonEventV1Type, - organizationId: string, -): Record => ({ - changed_at: toClickhouseTimestamp(event.changedAt), - organization_id: organizationId, - person_id: event.personId, - email: toNullableString(event.email), - is_archived: toFlag(event.isArchived), - merged_into_person_id: toNullableString(event.mergedIntoPersonId), - name: toNullableString(event.name), - primary_distinct_id: toNullableString(event.primaryDistinctId), - project_id: event.projectId, - traits: encodeJson(event.traits), - version: event.version, -}); - -export const toPersonIdentityRow = ( - event: ProcessorPersonIdentityEventV1Type, - organizationId: string, -): Record => ({ - changed_at: toClickhouseTimestamp(event.changedAt), - organization_id: organizationId, - person_id: event.personId, - distinct_id: event.distinctId, - is_deleted: toFlag(event.isDeleted), - previous_distinct_id: toNullableString(event.previousDistinctId), - project_id: event.projectId, - version: event.version, -}); - -export const toPendingOverrideRow = ( - event: ProcessorPersonIdentityEventV1Type, - organizationId: string, -): Record => ({ - changed_at: toClickhouseTimestamp(event.changedAt), - organization_id: organizationId, - person_id: event.personId, - is_deleted: toFlag(event.isDeleted), - project_id: event.projectId, - source_distinct_id: event.previousDistinctId ?? "", - target_distinct_id: event.distinctId, - version: event.version, -}); - -/** - * Builds the per-table ClickHouse row plan. Person/identity rows carry an - * `organization_id` resolved from their `project_id` via `organizationIdForProject` - * (the writer looks this up in MySQL); processed-event rows already carry the - * organization id from the upstream {@link ProcessedEventV2}. An unknown project - * resolves to `""`, which the readonly RLS user's row policy treats as not - * matching any tenant (fail-closed). - */ -export const buildAnalyticsWriterPlan = ( - messages: ReadonlyArray, - organizationIdForProject: (projectId: string) => string, -): AnalyticsWriterPlan => { - const processedEventRows: Array> = []; - const personRows: Array> = []; - const personIdentityRows: Array> = []; - const personIdentityOverrideRows: Array> = []; - const personIdentityPendingOverrideRows: Array> = []; - - for (const message of messages) { - switch (message.kind) { - case "processed": - processedEventRows.push(toProcessedEventRow(message.value)); - break; - case "person": - personRows.push( - toPersonRow(message.value, organizationIdForProject(message.value.projectId)), - ); - break; - case "person-distinct-id": { - const organizationId = organizationIdForProject(message.value.projectId); - personIdentityRows.push(toPersonIdentityRow(message.value, organizationId)); - if ( - typeof message.value.previousDistinctId === "string" && - message.value.previousDistinctId.length > 0 && - message.value.version > 0 - ) { - personIdentityOverrideRows.push(toPersonIdentityRow(message.value, organizationId)); - personIdentityPendingOverrideRows.push( - toPendingOverrideRow(message.value, organizationId), - ); - } - break; - } - } - } - - return { - personIdentityOverrideRows, - personIdentityPendingOverrideRows, - personIdentityRows, - personRows, - processedEventRows, - }; -}; - -// ============================================================================= -// Janitor snapshot resources -// ============================================================================= - -export interface BacklogRow { - readonly changed_at: string; - readonly person_id: string; - readonly project_id: string; - readonly source_distinct_id: string; - readonly target_distinct_id: string; - readonly version: number; -} - -export interface SnapshotResources { - readonly pendingOverrideDictionaryName: string; - readonly pendingOverrideSnapshotName: string; -} - -export const sanitizeIdentifier = (value: string): string => - value.replaceAll(/[^a-zA-Z0-9_]/g, "_"); - -// Names are unqualified — the runtime Clickhouse client connects with the -// per-stage database (provisioned by `Clickhouse.Database`) as its default, so -// the staging table created mid-squash lives in the right database. -export const makeSnapshotResources = (runId: string = createId()): SnapshotResources => { - const suffix = sanitizeIdentifier(runId.replaceAll("-", "")); - return { - pendingOverrideDictionaryName: `person_identity_pending_override_dict_${suffix}`, - pendingOverrideSnapshotName: `person_identity_pending_override_snapshot_${suffix}`, - }; -}; - -export const computeCutoffIso = ({ - now, - safetyWindowSeconds, -}: { - readonly now: Date; - readonly safetyWindowSeconds: number; -}): string => DateTime.formatIso(DateTime.makeUnsafe(now.getTime() - safetyWindowSeconds * 1000)); diff --git a/packages/core/src/domain/internalAnalytics/InternalAnalyticsEvents.ts b/packages/core/src/domain/internalAnalytics/InternalAnalyticsEvents.ts index a4769e8de..1506ad083 100644 --- a/packages/core/src/domain/internalAnalytics/InternalAnalyticsEvents.ts +++ b/packages/core/src/domain/internalAnalytics/InternalAnalyticsEvents.ts @@ -2,9 +2,8 @@ * Server-trusted analytics events emitted by internal services into the * analytics pipeline. The schema is a discriminated union of strict per-event * variants, discriminated on the wire-stable `eventName` literal. The same - * `eventName` is what downstream consumers (analytics-writer ClickHouse insert) - * see, so the literal serves as both the tagged-union discriminator and the - * wire identifier. + * `eventName` is preserved by every analytics store, so the literal serves as + * both the tagged-union discriminator and the wire identifier. * * Adding a new event: * 1. Define a `Schema.Struct` for the variant with a unique `eventName` diff --git a/packages/core/src/services/analytics/AnalyticsEventStore.ts b/packages/core/src/services/analytics/AnalyticsEventStore.ts new file mode 100644 index 000000000..c858ca997 --- /dev/null +++ b/packages/core/src/services/analytics/AnalyticsEventStore.ts @@ -0,0 +1,134 @@ +import { analyticsEvents, and, asc, Db, desc, gt, gte, inArray, lte } from "@voidhash/db"; +import { Context, Effect, Layer, Schema } from "effect"; + +import type { AnalyticsEventV1 } from "../../domain/analytics/AnalyticsEvent.ts"; + +export class AnalyticsEventStoreError extends Schema.TaggedErrorClass( + "AnalyticsEventStoreError", +)("AnalyticsEventStoreError", { + cause: Schema.String, + message: Schema.String, +}) {} + +export interface ListAnalyticsEventsInput { + readonly afterSequence?: number; + readonly end?: Date; + readonly eventNames?: ReadonlyArray; + readonly limit?: number; + readonly projectIds: ReadonlyArray; + readonly order?: "asc" | "desc"; + readonly start?: Date; +} + +export interface StoredAnalyticsEvent extends AnalyticsEventV1 { + readonly sequence: number; +} + +export interface AnalyticsEventStoreShape { + readonly insert: ( + events: ReadonlyArray, + ) => Effect.Effect; + readonly list: ( + input: ListAnalyticsEventsInput, + ) => Effect.Effect, AnalyticsEventStoreError>; +} + +const storedIdentityMode = (value: string): StoredAnalyticsEvent["identityMode"] => { + if (value === "full") return "full"; + return "personless"; +}; + +const storedSource = (value: string): StoredAnalyticsEvent["source"] => { + if (value === "revenue" || value === "internal") return value; + return "sdk"; +}; + +const toStoredEvent = (row: typeof analyticsEvents.$inferSelect): StoredAnalyticsEvent => ({ + schemaVersion: 1, + sequence: row.sequence, + eventId: row.eventId, + captureId: row.captureId, + eventName: row.eventName, + eventTimestamp: row.eventTimestamp, + processedAt: row.processedAt, + organizationId: row.organizationId, + projectId: row.projectId, + distinctId: row.distinctId, + previousDistinctId: row.previousDistinctId, + personId: row.personId, + identityMode: storedIdentityMode(row.identityMode), + properties: row.properties, + context: row.context, + sessionId: row.sessionId, + token: row.token, + requestId: row.requestId, + requestPath: row.requestPath, + source: storedSource(row.source), + sourceTopic: row.sourceTopic, +}); + +const makeAnalyticsEventStore = Effect.gen(function* () { + const db = yield* Db; + + const insert = (events: ReadonlyArray) => { + if (events.length === 0) return Effect.succeed(0); + return db + .insert(analyticsEvents) + .values([...events]) + .onConflictDoNothing({ target: [analyticsEvents.projectId, analyticsEvents.eventId] }) + .returning({ sequence: analyticsEvents.sequence }) + .pipe( + Effect.map((rows) => rows.length), + Effect.mapError( + (error) => + new AnalyticsEventStoreError({ + cause: String(error.cause), + message: "failed to insert analytics events", + }), + ), + ); + }; + + const list = (input: ListAnalyticsEventsInput) => { + if (input.projectIds.length === 0) return Effect.succeed([]); + const conditions = [inArray(analyticsEvents.projectId, [...input.projectIds])]; + if (input.afterSequence !== undefined) { + conditions.push(gt(analyticsEvents.sequence, input.afterSequence)); + } + if (input.start !== undefined) + conditions.push(gte(analyticsEvents.eventTimestamp, input.start)); + if (input.end !== undefined) conditions.push(lte(analyticsEvents.eventTimestamp, input.end)); + if (input.eventNames !== undefined && input.eventNames.length > 0) { + conditions.push(inArray(analyticsEvents.eventName, [...input.eventNames])); + } + let orderBy = asc(analyticsEvents.sequence); + if (input.order === "desc") orderBy = desc(analyticsEvents.sequence); + return db + .select() + .from(analyticsEvents) + .where(and(...conditions)) + .orderBy(orderBy) + .limit(input.limit ?? 10_000) + .pipe( + Effect.map((rows) => rows.map(toStoredEvent)), + Effect.mapError( + (error) => + new AnalyticsEventStoreError({ + cause: String(error.cause), + message: "failed to list analytics events", + }), + ), + ); + }; + + return { insert, list } satisfies AnalyticsEventStoreShape; +}); + +/** PostgreSQL implementation of the portable analytics event store. */ +export class AnalyticsEventStore extends Context.Service< + AnalyticsEventStore, + AnalyticsEventStoreShape +>()("@voidhash/core/AnalyticsEventStore") { + static readonly layer: Layer.Layer = + Layer.effect(AnalyticsEventStore)(makeAnalyticsEventStore); +} diff --git a/packages/core/src/services/analytics/AnalyticsService.ts b/packages/core/src/services/analytics/AnalyticsService.ts index c1f28c514..e05bc0f04 100644 --- a/packages/core/src/services/analytics/AnalyticsService.ts +++ b/packages/core/src/services/analytics/AnalyticsService.ts @@ -1,18 +1,9 @@ -/** - * `AnalyticsService` orchestrates the read-side analytics surface: the - * recent-events feed (`listRecentEvents`) and the multi-insight composition - * (`queryAnalyticsInsights`). Project lookups use `Db` inline; time-series and - * event reads use {@link ClickhouseWebClient}. The filter compiler, time-range - * resolver, and insight registry live in the analytics domain; the ClickHouse - * query accessor and metric-derivation resolver live alongside this file. A - * runtime without a ClickHouse service keeps authorization and Postgres scope - * checks but returns empty analytics results. - */ +import { Db } from "@voidhash/db"; import { constant, pick } from "@voidhash/lib/lang"; -import { Context, DateTime, Duration, Effect, Layer, Option, Schema } from "effect"; +import type { ListRecentAnalyticsEventsResponseType } from "@voidhash/rpc"; +import { Context, Effect, Layer, Schema } from "effect"; import { - type AnalyticsDataPoint, type AnalyticsFilter, type AnalyticsInsightQuery, type AnalyticsInsightResult, @@ -31,32 +22,15 @@ import { sumDataPoints, } from "../../domain/analytics/Analytics.ts"; import { AuthSession } from "../../domain/auth/Auth.ts"; -import { Db } from "@voidhash/db"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; import { checkOrganizationPermission, checkProjectPermission } from "../../utils/permissions.ts"; -import { RESERVED_REVENUE_EVENT_NAMES } from "../../domain/internalAnalytics/InternalAnalyticsEvents.ts"; -import { - CLICKHOUSE_EVENTS_FULL_TABLE, - CLICKHOUSE_PERSONS_FULL_TABLE, - analyticsAccessor, - getExperimentResults as getExperimentResultsQuery, -} from "./clickhouse-accessor.ts"; -import { buildSeriesResolver } from "./series-resolver.ts"; +import { AnalyticsEventStore } from "./AnalyticsEventStore.ts"; +import { resolvePostgresAnalyticsSeries } from "./postgres-series-resolver.ts"; const DEFAULT_LIMIT = 100; - -/** - * Conversion event assumed for an experiment that has not picked a primary - * metric — the conventional purchase event the paywall SDK bridge emits, which - * is the metric virtually every paywall test is measuring anyway. - */ -const DEFAULT_PRIMARY_METRIC_EVENT_NAME = "purchase_completed"; const MAX_LIMIT = 500; +const COMMUNITY_QUERY_EVENT_LIMIT = 100_000; -/** - * Catch-all service error. Wraps `SqlError`, `DbError`, and other - * infrastructural failures at the public-method boundary. - */ +/** Catch-all error for PostgreSQL analytics reads. */ export class AnalyticsServiceError extends Schema.TaggedErrorClass( "AnalyticsServiceError", )("AnalyticsServiceError", { cause: Schema.String, message: Schema.String }) {} @@ -74,282 +48,147 @@ export interface QueryAnalyticsInsightsInput { }>; } -interface RecentAnalyticsEventRow { - capture_id: string; - context: string; - event_id: string; - event_name: string; - event_properties: string; - identity_mode: string; - person_distinct_id: string | null; - person_email: string | null; - person_id: string | null; - person_name: string | null; - previous_distinct_id: string | null; - processed_at: string | Date; - received_at: string | Date; - request_id: string; +export interface ExperimentAnalyticsVariant { + readonly conversionRate: number; + readonly conversions: number; + readonly exposures: number; + readonly revenueUsd: number; + readonly variantKey: string; } -/** Conversion rate, guarding the zero-exposure division. */ -const conversionRateOf = (conversions: number, exposures: number): number => { - if (exposures > 0) return conversions / exposures; - return 0; -}; - -const parseDate = (value: string | Date): Date => { - if (value instanceof Date) return value; - const trimmed = value.trim(); - let normalized = trimmed; - if (!normalized.includes("T")) normalized = normalized.replace(" ", "T"); - // ClickHouse renders naive timestamps; anchor them to UTC before parsing. - if (!/(?:Z|[+-]\d{2}:\d{2})$/.test(normalized)) normalized = `${normalized}Z`; - return DateTime.toDateUtc(DateTime.makeUnsafe(normalized)); -}; - -/** JSON object payloads only: arrays and scalars decode to `None`, as before. */ -const decodeJsonRecord = Schema.decodeUnknownOption( - Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), -); - -const parseJsonRecord = (value: string): Record => { - if (!value || value.trim().length === 0) return {}; - return Option.getOrElse(decodeJsonRecord(value), () => ({})); -}; - +/** + * Community analytics reads over the portable PostgreSQL event log. The + * service keeps the existing built-in insight contract and deliberately omits + * custom insights, dashboards, cohorts, and VoidQL. + */ export class AnalyticsService extends Context.Service()("AnalyticsService", { make: Effect.gen(function* () { - const ch = Option.getOrUndefined( - yield* Effect.serviceOption(ClickhouseWebClient.ClickhouseWebClient), - ); const db = yield* Db; + const eventStore = yield* AnalyticsEventStore; const listRecentEvents = Effect.fn("analytics.listRecentEvents")( function* (input: { readonly limit?: number; readonly projectId: string }) { const session = yield* AuthSession; - yield* Effect.annotateCurrentSpan("voidhash.project.id", input.projectId); - if (session?.user?.id) - yield* Effect.annotateCurrentSpan("voidhash.user.id", session.user.id); yield* checkProjectPermission( input.projectId, "project:all", `User ${session?.user?.id} is not authorized to access analytics events for project ${input.projectId}`, ); - - if (ch === undefined) { - return { events: [], hasMore: false }; - } - const limit = Math.min(Math.max(input.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT); - yield* Effect.annotateCurrentSpan("voidhash.analytics.limit", limit); - - // The readonly ClickHouse user's row policies are keyed on - // `SQL_organization_id`, so resolve the project's org and pass it on - // every query (events_v2 + the persons_v1 join are both policy-scoped). - const project = yield* db.query.projects.findFirst({ - columns: { organizationId: true }, - where: { id: input.projectId }, + const rows = yield* eventStore.list({ + limit: limit + 1, + order: "desc", + projectIds: [input.projectId], }); - const organizationId = project?.organizationId ?? ""; - if (organizationId) - yield* Effect.annotateCurrentSpan("voidhash.organization.id", organizationId); - - // Structural interpolations (table names) are injected with `ch.literal` - // (raw SQL); scalar parameters use `ch.param` with their exact ClickHouse - // type so the readonly user's row policies / parser behave as before. - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - events.event_id AS event_id, - events.event_name AS event_name, - events.capture_id AS capture_id, - events.distinct_id AS person_distinct_id, - events.previous_distinct_id AS previous_distinct_id, - events.person_id AS person_id, - events.identity_mode AS identity_mode, - events.request_id AS request_id, - events.event_ts AS received_at, - events.processed_ts AS processed_at, - events.event_properties AS event_properties, - events.context AS context, - persons.name AS person_name, - persons.email AS person_email - FROM ${ch.literal(CLICKHOUSE_EVENTS_FULL_TABLE)} AS events - LEFT JOIN ( - SELECT - analytics_persons.person_id AS person_id, - argMax(analytics_persons.email, analytics_persons.version) AS email, - argMax(analytics_persons.name, analytics_persons.version) AS name - FROM ${ch.literal(CLICKHOUSE_PERSONS_FULL_TABLE)} AS analytics_persons - WHERE analytics_persons.project_id = ${ch.param("String", input.projectId)} - AND analytics_persons.is_archived = 0 - GROUP BY analytics_persons.person_id - ) AS persons ON events.person_id = persons.person_id - WHERE events.project_id = ${ch.param("String", input.projectId)} - ORDER BY events.event_ts DESC, events.event_id DESC - LIMIT ${ch.param("UInt32", limit + 1)} - `, - { SQL_organization_id: organizationId }, - ); - - yield* Effect.annotateCurrentSpan("voidhash.analytics.has_more", rows.length > limit); - - return { + const response: ListRecentAnalyticsEventsResponseType = { events: rows.slice(0, limit).map((row) => ({ - captureId: row.capture_id, - context: parseJsonRecord(row.context), - eventId: row.event_id, - eventName: row.event_name, - identityMode: row.identity_mode, - personDistinctId: row.person_distinct_id, - personEmail: row.person_email, - personId: row.person_id, - personName: row.person_name, - previousDistinctId: row.previous_distinct_id, - processedAt: parseDate(row.processed_at), - properties: parseJsonRecord(row.event_properties), - receivedAt: parseDate(row.received_at), - requestId: row.request_id, + captureId: row.captureId, + context: row.context, + eventId: row.eventId, + eventName: row.eventName, + identityMode: row.identityMode, + personDistinctId: row.distinctId, + personEmail: null, + personId: row.personId, + personName: null, + previousDistinctId: row.previousDistinctId, + processedAt: row.processedAt, + properties: row.properties, + receivedAt: row.eventTimestamp, + requestId: row.requestId, })), hasMore: rows.length > limit, }; + return response; }, (effect) => effect.pipe( - Effect.catchTags({ - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to list recent analytics events", - }), - ), - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to list recent analytics events", - }), - ), - }), + Effect.catchTag("AnalyticsEventStoreError", (error) => + Effect.fail(new AnalyticsServiceError({ cause: error.cause, message: error.message })), + ), ), ); const queryAnalyticsInsights = Effect.fn("queryAnalyticsInsights")( function* (input: QueryAnalyticsInsightsInput) { const session = yield* AuthSession; - if (session?.user?.id) - yield* Effect.annotateCurrentSpan("voidhash.user.id", session.user.id); - yield* Effect.annotateCurrentSpan("voidhash.analytics.query_count", input.queries.length); - const { getSeries } = buildSeriesResolver(analyticsAccessor); - - const results: ReadonlyArray<{ + const results: Array<{ insightId: BuiltInInsightId; key: string; resolvedTimeRange: { start: Date; end: Date }; result: AnalyticsInsightResult; - }>[number][] = []; + }> = []; for (const query of input.queries) { - if (query.context.organizationId) - yield* Effect.annotateCurrentSpan( - "voidhash.organization.id", - query.context.organizationId, - ); - yield* Effect.annotateCurrentSpan("voidhash.analytics.insight_id", query.insightId); - yield* Effect.annotateCurrentSpan("voidhash.analytics.insight_key", query.key); - yield* checkOrganizationPermission( query.context.organizationId, "organization:all", `User ${session?.user?.id} is not authorized to access analytics for organization ${query.context.organizationId}`, ); - const insight = yield* getBuiltInInsight(query.insightId); yield* ensureNoBreakdowns(query.breakdowns); - const resolvedTimeRange = yield* resolveTimeRange(query.timeRange); - yield* Effect.annotateCurrentSpan( - "voidhash.analytics.time_range.start", - resolvedTimeRange.start.toISOString(), - ); - yield* Effect.annotateCurrentSpan( - "voidhash.analytics.time_range.end", - resolvedTimeRange.end.toISOString(), - ); const projectRows = yield* db.query.projects.findMany({ + columns: { id: true }, where: { organizationId: query.context.organizationId }, }); - const availableProjectIds = projectRows.map((project) => project.id); const compiledFilter: CompiledAnalyticsFilter = yield* compileAnalyticsFilter({ - availableProjectIds, + availableProjectIds: projectRows.map((project) => project.id), filter: query.filter, supportedFields: insight.supportedFilterFields, }); - yield* Effect.annotateCurrentSpan( - "voidhash.analytics.project_ids.count", - compiledFilter.projectIds.length, - ); const granularity = query.granularity ?? insight.defaultGranularity; - yield* Effect.annotateCurrentSpan("voidhash.analytics.granularity", granularity); - if (!insight.supportedGranularities.includes(granularity)) { - yield* Effect.fail( + return yield* Effect.fail( new InvalidAnalyticsQueryError({ message: `Granularity ${granularity} is not supported for ${query.insightId}`, }), ); } - let series: AnalyticsDataPoint[] = []; - if (compiledFilter.projectIds.length && ch !== undefined) { - series = yield* getSeries( - query.insightId, - compiledFilter, - granularity, - resolvedTimeRange, - query.context.organizationId, - ).pipe(Effect.provideService(ClickhouseWebClient.ClickhouseWebClient, ch)); - } - + const events = yield* eventStore.list({ + end: resolvedTimeRange.end, + limit: COMMUNITY_QUERY_EVENT_LIMIT, + projectIds: compiledFilter.projectIds, + }); + const series = resolvePostgresAnalyticsSeries({ + end: resolvedTimeRange.end, + events, + filters: compiledFilter, + granularity, + insightId: query.insightId, + start: resolvedTimeRange.start, + }); let summaryValue = sumDataPoints(series); - if (RATE_INSIGHTS.has(query.insightId)) { - summaryValue = avgDataPoints(series); - } - - const result: AnalyticsInsightResult = { - kind: "metric", - sparkline: series, - summary: { - currency: pick(CURRENCY_INSIGHTS.has(query.insightId), "USD", undefined), - value: summaryValue, - }, - }; + if (RATE_INSIGHTS.has(query.insightId)) summaryValue = avgDataPoints(series); results.push({ insightId: query.insightId, key: query.key, resolvedTimeRange, - result, + result: { + kind: "metric", + sparkline: series, + summary: { + currency: pick(CURRENCY_INSIGHTS.has(query.insightId), "USD", undefined), + value: summaryValue, + }, + }, }); } - return { results }; }, (effect) => effect.pipe( Effect.catchTags({ - EffectDrizzleQueryError: (error) => + AnalyticsEventStoreError: (error) => Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to query analytics insights", - }), + new AnalyticsServiceError({ cause: error.cause, message: error.message }), ), - SqlError: (error) => + EffectDrizzleQueryError: (error) => Effect.fail( new AnalyticsServiceError({ - cause: error.message, + cause: String(error.cause), message: "Failed to query analytics insights", }), ), @@ -357,19 +196,10 @@ export class AnalyticsService extends Context.Service()("Analy ), ); - /** - * Raw per-variant results for an experiment: exposures, conversions (primary - * metric fired after first exposure), conversion rate, and post-exposure - * revenue. Delegates the ClickHouse funnel to the accessor - * ({@link getExperimentResultsQuery}); resolves the experiment's project + - * organization (for the row policy) and its metric names here, falling back - * to {@link DEFAULT_PRIMARY_METRIC_EVENT_NAME} when none was chosen. Reads - * server-emitted `$experiment.exposed` events — returns zeroed variants until - * exposure emission is live. - */ const getExperimentResults = Effect.fn("analytics.getExperimentResults")( function* (input: { readonly experimentId: string; readonly days?: number }) { const experiment = yield* db.query.experiments.findFirst({ + columns: { projectId: true }, where: { id: input.experimentId }, }); if (!experiment) { @@ -385,80 +215,25 @@ export class AnalyticsService extends Context.Service()("Analy "project:all", `Not authorized to read experiment results for ${input.experimentId}`, ); - const project = yield* db.query.projects.findFirst({ - where: { id: experiment.projectId }, - }); - if (!project) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: experiment.projectId, - message: "Project not found", - }), - ); - } - if (ch === undefined) { - return { variants: [] }; - } - const nowInstant = yield* DateTime.now; - const now = DateTime.toDateUtc(nowInstant); - const startDate = - experiment.startedAt ?? - DateTime.toDateUtc( - DateTime.subtractDuration(nowInstant, Duration.days(input.days ?? 90)), - ); - const endDate = experiment.endedAt ?? now; - - const rows = yield* getExperimentResultsQuery({ - organizationId: project.organizationId, - projectId: experiment.projectId, - experimentId: experiment.id, - primaryMetricEventNames: [ - experiment.primaryMetricEventName ?? DEFAULT_PRIMARY_METRIC_EVENT_NAME, - ], - revenueEventNames: [...RESERVED_REVENUE_EVENT_NAMES], - startDate, - endDate, - }).pipe(Effect.provideService(ClickhouseWebClient.ClickhouseWebClient, ch)); - - return { - variants: rows.map((r) => { - const exposures = Number(r.exposures); - const conversions = Number(r.conversions); - return { - variantKey: r.variant, - exposures, - conversions, - conversionRate: conversionRateOf(conversions, exposures), - revenueUsd: Number(r.revenue_cents) / 100, - }; - }), - }; + const variants: ExperimentAnalyticsVariant[] = []; + return { variants }; }, (effect) => effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to query experiment results", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to query experiment results", - }), - ), - }), + Effect.catchTag("EffectDrizzleQueryError", (error) => + Effect.fail( + new AnalyticsServiceError({ + cause: String(error.cause), + message: "Failed to query experiment results", + }), + ), + ), ), ); return constant({ getExperimentResults, listRecentEvents, queryAnalyticsInsights }); }), }) { - static layer: Layer.Layer = Layer.effect( - AnalyticsService, - )(AnalyticsService.make); + static readonly layer: Layer.Layer = + Layer.effect(AnalyticsService)(AnalyticsService.make); } diff --git a/packages/core/src/services/analytics/CustomAnalyticsService.ts b/packages/core/src/services/analytics/CustomAnalyticsService.ts deleted file mode 100644 index 2b868d1c2..000000000 --- a/packages/core/src/services/analytics/CustomAnalyticsService.ts +++ /dev/null @@ -1,2745 +0,0 @@ -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { - Db, - analyticsCohortMembers, - analyticsCohorts, - analyticsDashboardItems, - analyticsDashboards, - analyticsInsights, - analyticsSavedQuery, - and, - asc, - desc, - eq, - inArray, - isNull, - persons, -} from "@voidhash/db"; -import { constant } from "@voidhash/lib/lang"; -import type { - AnalyticsActorType, - AnalyticsCohortType, - AnalyticsDashboardItemLayoutType, - AnalyticsDashboardType, - AnalyticsFilterType, - AnalyticsTrendsComparisonType, - AnalyticsTrendsFormulaType, - CustomAnalyticsInsightQueryType, - QueryCustomAnalyticsInsightResponseType, - SavedAnalyticsInsightType, - SavedVoidQlInsightType, -} from "@voidhash/rpc"; -import { Context, DateTime, Effect, Layer, Option } from "effect"; - -import { InvalidAnalyticsQueryError, resolveTimeRange } from "../../domain/analytics/Analytics.ts"; -import { AuthSession } from "../../domain/auth/Auth.ts"; -import { generateId } from "../../utils/generate-id.ts"; -import { checkProjectPermission } from "../../utils/permissions.ts"; -import { AnalyticsServiceError } from "./AnalyticsService.ts"; -import { - type EventLifecyclePoint, - type EventPathLink, - type EventRetentionCohort, - type EventStickinessBucket, - getEventFunnelBreakdownCounts, - getEventFunnelCounts, - getEventLifecyclePoints, - getEventPathLinks, - getEventPersonDrilldown, - getEventRetentionCohorts, - getEventStickinessBuckets, - getEventTrendSeries, -} from "./clickhouse-accessor.ts"; - -type AnalyticsInsightRow = typeof analyticsInsights.$inferSelect; -type AnalyticsDashboardRow = typeof analyticsDashboards.$inferSelect; -type AnalyticsCohortRow = typeof analyticsCohorts.$inferSelect; -type AnalyticsSavedQueryRow = typeof analyticsSavedQuery.$inferSelect; - -/** Builds a `Date` without the ambient constructor, from epoch millis or another date. */ -const dateFrom = (input: Date | number): Date => DateTime.toDateUtc(DateTime.makeUnsafe(input)); - -/** Wall-clock timestamp used for row bookkeeping columns. */ -const currentTimestamp = (): Date => DateTime.toDateUtc(DateTime.nowUnsafe()); - -const CUSTOM_INSIGHT_KINDS = new Set([ - "trends", - "funnels", - "retention", - "paths", - "stickiness", - "lifecycle", -]); - -/** - * Narrows the untyped `jsonb` definition column to the saved insight query union. - * - * Rows are only ever written from an already validated RPC payload, so the tag - * check is enough to trust the stored shape. - */ -const isCustomAnalyticsInsightQuery = ( - value: unknown, -): value is CustomAnalyticsInsightQueryType => - typeof value === "object" && - value !== null && - "kind" in value && - typeof value.kind === "string" && - CUSTOM_INSIGHT_KINDS.has(value.kind); - -const toSavedInsight = (row: AnalyticsInsightRow): Effect.Effect => - Effect.gen(function* () { - if (!isCustomAnalyticsInsightQuery(row.definition)) { - return yield* Effect.die(`Analytics insight ${row.id} has a malformed definition`); - } - const definition = row.definition; - return { - createdAt: row.createdAt, - createdBy: row.createdBy, - definition, - description: row.description, - id: row.id, - kind: definition.kind, - name: row.name, - organizationId: row.organizationId, - projectId: row.projectId, - updatedAt: row.updatedAt, - }; - }); - -const toSavedVoidQlInsight = (row: AnalyticsSavedQueryRow): SavedVoidQlInsightType => ({ - createdAt: row.createdAt, - createdBy: row.createdBy, - id: row.id, - name: row.name, - organizationId: row.organizationId, - schemaVersion: row.schemaVersion, - text: row.voidqlText, - updatedAt: row.updatedAt, -}); - -export type ExecutableTrendsDefinition = Extract< - CustomAnalyticsInsightQueryType, - { readonly kind: "trends" } ->; -export type ExecutableFunnelsDefinition = Extract< - CustomAnalyticsInsightQueryType, - { readonly kind: "funnels" } ->; -export type ExecutableRetentionDefinition = Extract< - CustomAnalyticsInsightQueryType, - { readonly kind: "retention" } ->; -export type ExecutablePathsDefinition = Extract< - CustomAnalyticsInsightQueryType, - { readonly kind: "paths" } ->; -export type ExecutableStickinessDefinition = Extract< - CustomAnalyticsInsightQueryType, - { readonly kind: "stickiness" } ->; -export type ExecutableLifecycleDefinition = Extract< - CustomAnalyticsInsightQueryType, - { readonly kind: "lifecycle" } ->; - -type TrendsInsightResult = Extract< - QueryCustomAnalyticsInsightResponseType, - { readonly kind: "trends" } ->; -type ResolvedTrendsTimeRange = TrendsInsightResult["resolvedTimeRange"]; - -const shiftUtcYear = (value: Date, years: number): Date => { - const targetYear = value.getUTCFullYear() + years; - const lastDay = dateFrom(Date.UTC(targetYear, value.getUTCMonth() + 1, 0)).getUTCDate(); - return dateFrom( - Date.UTC( - targetYear, - value.getUTCMonth(), - Math.min(value.getUTCDate(), lastDay), - value.getUTCHours(), - value.getUTCMinutes(), - value.getUTCSeconds(), - ), - ); -}; - -/** Resolve the date window used for a Trends comparison. */ -export const resolveTrendsComparisonTimeRange = ( - comparison: AnalyticsTrendsComparisonType, - current: ResolvedTrendsTimeRange, -): ResolvedTrendsTimeRange => { - if (comparison === "previous_year") { - return { end: shiftUtcYear(current.end, -1), start: shiftUtcYear(current.start, -1) }; - } - - const duration = current.end.getTime() - current.start.getTime(); - return { - end: dateFrom(current.start.getTime() - 1_000), - start: dateFrom(current.start.getTime() - duration), - }; -}; - -/** Resolve a Trends comparison window when the definition asks for one. */ -const resolveOptionalTrendsComparisonTimeRange = ( - comparison: AnalyticsTrendsComparisonType | undefined, - current: ResolvedTrendsTimeRange, -): ResolvedTrendsTimeRange | undefined => { - if (comparison === undefined) return undefined; - return resolveTrendsComparisonTimeRange(comparison, current); -}; - -/** Key suffix that distinguishes a comparison period's series from the current one. */ -const trendsComparisonKeySuffix = ( - comparison: "current" | AnalyticsTrendsComparisonType, -): string => { - if (comparison === "current") return ""; - return `:comparison:${comparison}`; -}; - -/** Human-readable name for a Trends comparison period. */ -const trendsComparisonLabel = ( - comparison: "current" | AnalyticsTrendsComparisonType, -): string | undefined => { - if (comparison === "previous_period") return "previous period"; - if (comparison === "previous_year") return "previous year"; - return undefined; -}; - -const labelWithComparison = ( - label: string, - comparison: "current" | AnalyticsTrendsComparisonType, -): string => { - const suffix = trendsComparisonLabel(comparison); - if (suffix === undefined) return label; - return `${label} (${suffix})`; -}; - -const keyWithBreakdown = (key: string, breakdownValue: string | undefined): string => { - if (breakdownValue === undefined) return key; - return `${key}:${breakdownValue}`; -}; - -const labelWithBreakdown = (label: string, breakdownValue: string | undefined): string => { - if (breakdownValue === undefined) return label; - return `${label} · ${breakdownValue || "(empty)"}`; -}; - -const stripSuffix = (key: string, suffix: string): string => { - if (suffix.length === 0) return key; - return key.slice(0, -suffix.length); -}; - -const breakdownValueFromKey = (sourceKey: string, seriesKey: string): string | undefined => { - if (sourceKey === seriesKey) return undefined; - return sourceKey.slice(seriesKey.length + 1); -}; - -/** Divide while treating an empty denominator as a zero rate. */ -const safeRatio = (numerator: number, denominator: number): number => { - if (denominator === 0) return 0; - return numerator / denominator; -}; - -const startOfTrendsBucket = ( - value: Date, - granularity: ExecutableTrendsDefinition["granularity"], -): Date => { - const bucket = dateFrom(value); - bucket.setUTCMilliseconds(0); - if (granularity === "hour") bucket.setUTCMinutes(0, 0, 0); - else { - bucket.setUTCHours(0, 0, 0, 0); - if (granularity === "week") { - bucket.setUTCDate(bucket.getUTCDate() - ((bucket.getUTCDay() + 6) % 7)); - } else if (granularity === "month") bucket.setUTCDate(1); - else if (granularity === "quarter") { - bucket.setUTCMonth(Math.floor(bucket.getUTCMonth() / 3) * 3, 1); - } else if (granularity === "year") bucket.setUTCMonth(0, 1); - } - return bucket; -}; - -const nextTrendsBucket = ( - value: Date, - granularity: ExecutableTrendsDefinition["granularity"], -): Date => { - const next = dateFrom(value); - if (granularity === "hour") next.setUTCHours(next.getUTCHours() + 1); - else if (granularity === "day") next.setUTCDate(next.getUTCDate() + 1); - else if (granularity === "week") next.setUTCDate(next.getUTCDate() + 7); - else if (granularity === "month") next.setUTCMonth(next.getUTCMonth() + 1); - else if (granularity === "quarter") next.setUTCMonth(next.getUTCMonth() + 3); - else next.setUTCFullYear(next.getUTCFullYear() + 1); - return next; -}; - -const trendsBuckets = ( - range: ResolvedTrendsTimeRange, - granularity: ExecutableTrendsDefinition["granularity"], -): Date[] => { - const buckets: Date[] = []; - let cursor = startOfTrendsBucket(range.start, granularity); - while (cursor <= range.end && buckets.length < 20_000) { - buckets.push(cursor); - cursor = nextTrendsBucket(cursor, granularity); - } - return buckets; -}; - -/** Align comparison points to the current x-axis by bucket position. */ -export const alignTrendsComparisonPoints = ( - points: TrendsInsightResult["series"][number]["points"], - current: ResolvedTrendsTimeRange, - comparison: ResolvedTrendsTimeRange, - granularity: ExecutableTrendsDefinition["granularity"], -): TrendsInsightResult["series"][number]["points"] => { - const currentBuckets = trendsBuckets(current, granularity); - const comparisonBuckets = trendsBuckets(comparison, granularity); - const currentByComparisonTimestamp = new Map( - comparisonBuckets.map((bucket, index) => [bucket.getTime(), currentBuckets[index]]), - ); - return points.flatMap((point) => { - const timestamp = currentByComparisonTimestamp.get(point.timestamp.getTime()); - if (timestamp === undefined) return []; - return [{ ...point, timestamp }]; - }); -}; - -/** Fill absent Trends buckets with zero so charts and formulas share a complete time axis. */ -export const fillTrendsSeriesPoints = ( - points: TrendsInsightResult["series"][number]["points"], - range: ResolvedTrendsTimeRange, - granularity: ExecutableTrendsDefinition["granularity"], -): TrendsInsightResult["series"][number]["points"] => { - const values = new Map(points.map((point) => [point.timestamp.getTime(), point.value])); - return trendsBuckets(range, granularity).map((timestamp) => ({ - timestamp, - value: values.get(timestamp.getTime()) ?? 0, - })); -}; - -/** Apply trailing smoothing, cumulative values, and weekend removal to a complete Trends series. */ -export const applyTrendsPresentation = ( - points: TrendsInsightResult["series"][number]["points"], - options: { - readonly cumulative?: boolean; - readonly hideWeekends?: boolean; - readonly smoothingWindow?: number; - }, -): TrendsInsightResult["series"][number]["points"] => { - const smoothingWindow = Math.max(1, options.smoothingWindow ?? 1); - let presented = points.map((point, index) => { - if (smoothingWindow === 1) return point; - const window = points.slice(Math.max(0, index - smoothingWindow + 1), index + 1); - return { - ...point, - value: window.reduce((sum, candidate) => sum + candidate.value, 0) / window.length, - }; - }); - if (options.cumulative) { - let total = 0; - presented = presented.map((point) => { - total += point.value; - return { ...point, value: total }; - }); - } - if (options.hideWeekends) { - return presented.filter((point) => ![0, 6].includes(point.timestamp.getUTCDay())); - } - return presented; -}; - -type TrendsFormulaNode = - | { readonly kind: "number"; readonly value: number } - | { readonly key: string; readonly kind: "series" } - | { - readonly kind: "unary"; - readonly operand: TrendsFormulaNode; - readonly operator: "+" | "-"; - } - | { - readonly kind: "binary"; - readonly left: TrendsFormulaNode; - readonly operator: "+" | "-" | "*" | "/" | "%" | "**"; - readonly right: TrendsFormulaNode; - }; - -type TrendsFormulaToken = - | { readonly kind: "identifier"; readonly value: string } - | { readonly kind: "number"; readonly value: number } - | { readonly kind: "operator"; readonly value: "+" | "-" | "*" | "/" | "%" | "**" } - | { readonly kind: "left_parenthesis" | "right_parenthesis" }; - -const TRENDS_FORMULA_OPERATORS = constant(["+", "-", "*", "/", "%"]); - -const arithmeticOperator = ( - character: string | undefined, -): (typeof TRENDS_FORMULA_OPERATORS)[number] | undefined => - TRENDS_FORMULA_OPERATORS.find((operator) => operator === character); - -const tokenizeTrendsFormula = ( - source: string, -): Effect.Effect => - Effect.gen(function* () { - const tokens: TrendsFormulaToken[] = []; - let offset = 0; - while (offset < source.length) { - const character = source[offset]; - if (character && /\s/u.test(character)) { - offset += 1; - continue; - } - const remainder = source.slice(offset); - const number = /^(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?/iu.exec(remainder)?.[0]; - const identifier = /^[a-z][a-z0-9_]*/iu.exec(remainder)?.[0]; - const operator = arithmeticOperator(character); - if (number) { - const value = Number(number); - if (!Number.isFinite(value)) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Formula numbers must be finite" }), - ); - } - tokens.push({ kind: "number", value }); - offset += number.length; - } else if (identifier) { - tokens.push({ kind: "identifier", value: identifier.toLowerCase() }); - offset += identifier.length; - } else if (remainder.startsWith("**")) { - tokens.push({ kind: "operator", value: "**" }); - offset += 2; - } else if (operator !== undefined) { - tokens.push({ kind: "operator", value: operator }); - offset += 1; - } else if (character === "(") { - tokens.push({ kind: "left_parenthesis" }); - offset += 1; - } else if (character === ")") { - tokens.push({ kind: "right_parenthesis" }); - offset += 1; - } else { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `Unexpected character at position ${offset + 1}`, - }), - ); - } - if (tokens.length > 128) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Formula is too complex" }), - ); - } - } - return tokens; - }); - -/** Recursive-descent parser over the tokenized Trends formula grammar. */ -const parseTrendsFormulaTokens = ( - tokens: ReadonlyArray, -): Effect.Effect => { - let offset = 0; - - const current = (): TrendsFormulaToken | undefined => tokens[offset]; - - const consume = (): Effect.Effect => { - const token = current(); - if (!token) { - return Effect.fail( - new InvalidAnalyticsQueryError({ message: "Formula ended unexpectedly" }), - ); - } - offset += 1; - return Effect.succeed(token); - }; - - const parsePrimary = (): Effect.Effect => - Effect.gen(function* () { - const token = yield* consume(); - if (token.kind === "number") return { kind: "number", value: token.value }; - if (token.kind === "identifier") return { key: token.value, kind: "series" }; - if (token.kind === "left_parenthesis") { - const expression = yield* parseAdditive(); - const closing = yield* consume(); - if (closing.kind !== "right_parenthesis") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Formula has an unmatched parenthesis", - }), - ); - } - return expression; - } - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Expected a number, series, or parenthesized expression", - }), - ); - }); - - const parsePower = (): Effect.Effect => - Effect.gen(function* () { - const left = yield* parsePrimary(); - const token = current(); - if (token?.kind === "operator" && token.value === "**") { - yield* consume(); - return { kind: "binary", left, operator: "**", right: yield* parseUnary() }; - } - return left; - }); - - const parseUnary = (): Effect.Effect => - Effect.gen(function* () { - const token = current(); - if (token?.kind === "operator" && (token.value === "+" || token.value === "-")) { - yield* consume(); - return { kind: "unary", operand: yield* parseUnary(), operator: token.value }; - } - return yield* parsePower(); - }); - - const parseMultiplicative = (): Effect.Effect => - Effect.gen(function* () { - let left = yield* parseUnary(); - while (true) { - const operator = current(); - if ( - operator?.kind !== "operator" || - (operator.value !== "*" && operator.value !== "/" && operator.value !== "%") - ) { - break; - } - yield* consume(); - const right = yield* parseUnary(); - left = { - kind: "binary", - left, - operator: operator.value, - right, - }; - } - return left; - }); - - const parseAdditive = (): Effect.Effect => - Effect.gen(function* () { - let left = yield* parseMultiplicative(); - while (true) { - const operator = current(); - if (operator?.kind !== "operator" || (operator.value !== "+" && operator.value !== "-")) { - break; - } - yield* consume(); - const right = yield* parseMultiplicative(); - left = { - kind: "binary", - left, - operator: operator.value, - right, - }; - } - return left; - }); - - return Effect.gen(function* () { - if (tokens.length === 0) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Formula cannot be empty" }), - ); - } - const expression = yield* parseAdditive(); - if (current()) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Unexpected token after formula" }), - ); - } - return expression; - }); -}; - -const collectTrendsFormulaReferences = (node: TrendsFormulaNode, references: Set): void => { - if (node.kind === "series") references.add(node.key); - else if (node.kind === "unary") collectTrendsFormulaReferences(node.operand, references); - else if (node.kind === "binary") { - collectTrendsFormulaReferences(node.left, references); - collectTrendsFormulaReferences(node.right, references); - } -}; - -const applyTrendsFormulaOperator = ( - operator: Extract["operator"], - left: number, - right: number, -): number => { - if (operator === "+") return left + right; - if (operator === "-") return left - right; - if (operator === "*") return left * right; - if (operator === "/") { - if (right === 0) return 0; - return left / right; - } - if (operator === "%") { - if (right === 0) return 0; - return left % right; - } - return left ** right; -}; - -const evaluateTrendsFormulaNode = ( - node: TrendsFormulaNode, - values: ReadonlyMap, -): number => { - if (node.kind === "number") return node.value; - if (node.kind === "series") return values.get(node.key) ?? 0; - if (node.kind === "unary") { - const value = evaluateTrendsFormulaNode(node.operand, values); - if (node.operator === "-") return -value; - return value; - } - const left = evaluateTrendsFormulaNode(node.left, values); - const right = evaluateTrendsFormulaNode(node.right, values); - const result = applyTrendsFormulaOperator(node.operator, left, right); - if (Number.isFinite(result)) return result; - return 0; -}; - -const compileTrendsFormula = ( - formula: AnalyticsTrendsFormulaType, - allowedSeries: ReadonlySet, -): Effect.Effect => - Effect.gen(function* () { - const node = yield* tokenizeTrendsFormula(formula.expression).pipe( - Effect.flatMap(parseTrendsFormulaTokens), - Effect.mapError( - (error) => - new InvalidAnalyticsQueryError({ - message: `Invalid Trends formula ${formula.key}: ${error.message}`, - }), - ), - ); - const references = new Set(); - collectTrendsFormulaReferences(node, references); - const missing = [...references].filter((reference) => !allowedSeries.has(reference)); - if (missing.length > 0) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `Trends formula ${formula.key} references unknown series: ${missing.join(", ")}`, - }), - ); - } - return node; - }); - -const CUSTOM_PROPERTY_PREFIX = "event.properties."; -const CUSTOM_EVENT_FIELDS = new Set(["event.name", "person.id"]); -const PROPERTY_AGGREGATIONS = new Set([ - "property_sum", - "property_average", - "property_minimum", - "property_maximum", - "property_median", - "property_p75", - "property_p90", - "property_p95", - "property_p99", -]); - -const validateCustomEventField = ( - field: string, -): Effect.Effect => { - if (CUSTOM_EVENT_FIELDS.has(field)) return Effect.void; - if ( - field.startsWith(CUSTOM_PROPERTY_PREFIX) && - field.length > CUSTOM_PROPERTY_PREFIX.length && - field.length <= CUSTOM_PROPERTY_PREFIX.length + 128 && - !Array.from(field).some((character) => character.charCodeAt(0) < 32) - ) { - return Effect.void; - } - return Effect.fail( - new InvalidAnalyticsQueryError({ - message: `Unsupported custom analytics field: ${field}`, - }), - ); -}; - -const validateCustomEventFilter = ( - filter: AnalyticsFilterType, - depth = 0, -): Effect.Effect => - Effect.gen(function* () { - if (depth > 8) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Custom analytics filters are nested too deeply", - }), - ); - } - if (filter.type === "not") return yield* validateCustomEventFilter(filter.filter, depth + 1); - // The `and`/`or` group shares one union member, so it is narrowed by excluding - // the other tags rather than by testing its own tag. - if (filter.type !== "predicate") { - if (filter.filters.length === 0) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Custom analytics filter groups cannot be empty", - }), - ); - } - const counts = yield* Effect.all( - filter.filters.map((child) => validateCustomEventFilter(child, depth + 1)), - ); - return counts.reduce((total, count) => total + count, 0); - } - - const predicate = filter; - yield* validateCustomEventField(predicate.field); - if (["gt", "gte", "lt", "lte"].includes(predicate.op)) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `Custom analytics does not yet support the ${predicate.op} property operator`, - }), - ); - } - if (predicate.op === "exists") return 1; - if ((predicate.op === "in" || predicate.op === "not_in") && !Array.isArray(predicate.value)) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${predicate.op} requires an array value`, - }), - ); - } - if (predicate.value === undefined || predicate.value === null) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: `${predicate.op} requires a value` }), - ); - } - return 1; - }); - -/** Validate the currently executable subset of custom analytics definitions. */ -export const validateExecutableTrendsDefinition = ( - definition: CustomAnalyticsInsightQueryType, -): Effect.Effect => - Effect.gen(function* () { - if (definition.kind !== "trends") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${definition.kind} execution is not available yet`, - }), - ); - } - if (definition.breakdown) yield* validateCustomEventField(definition.breakdown.field); - if ( - definition.smoothingWindow !== undefined && - (!Number.isSafeInteger(definition.smoothingWindow) || definition.granularity !== "day") - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Trends smoothing requires a whole-day window between 1 and 28", - }), - ); - } - if (definition.hideWeekends && definition.granularity !== "day") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Weekend hiding requires daily granularity" }), - ); - } - if ( - definition.display === "number" && - (definition.cumulative || definition.hideWeekends || (definition.smoothingWindow ?? 1) > 1) - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Total-value Trends do not support time-series presentation options", - }), - ); - } - const seriesKeys = new Set(definition.series.map((series) => series.key.toLowerCase())); - if (seriesKeys.size !== definition.series.length) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Trends series keys must be unique" }), - ); - } - if (definition.formulas) { - if (definition.formulas.length === 0 || definition.formulas.length > 8) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Trends supports between 1 and 8 formulas" }), - ); - } - const formulaKeys = new Set(definition.formulas.map((formula) => formula.key.toLowerCase())); - if ( - formulaKeys.size !== definition.formulas.length || - [...formulaKeys].some((key) => seriesKeys.has(key)) - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Trends formula keys must be unique and distinct from series keys", - }), - ); - } - yield* Effect.all( - definition.formulas.map((formula) => compileTrendsFormula(formula, seriesKeys)), - { concurrency: 4 }, - ); - } - let predicateCount = 0; - for (const series of definition.series) { - if (PROPERTY_AGGREGATIONS.has(series.aggregation)) { - if (!series.mathProperty?.trim()) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${series.aggregation} requires an event property`, - }), - ); - } - yield* validateCustomEventField(`${CUSTOM_PROPERTY_PREFIX}${series.mathProperty.trim()}`); - } else if (series.mathProperty !== undefined) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${series.aggregation} does not use an event property`, - }), - ); - } - if (series.filters) predicateCount += yield* validateCustomEventFilter(series.filters); - } - if (predicateCount > 20) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Custom Trends supports at most 20 predicates" }), - ); - } - return definition; - }); - -interface TrendsFormulaGroup { - readonly breakdownValue?: string; - readonly comparison: "current" | AnalyticsTrendsComparisonType; - readonly series: Map>; -} - -/** Build formula-only Trends series from the queried source series and their aligned buckets. */ -export const buildTrendsFormulaSeries = ( - definition: ExecutableTrendsDefinition, - sourceSeries: ReadonlyArray, -): Effect.Effect => - Effect.gen(function* () { - if (!definition.formulas?.length) return [...sourceSeries]; - const allowedSeries = new Set(definition.series.map((series) => series.key.toLowerCase())); - const compiled = yield* Effect.all( - definition.formulas.map((formula) => compileTrendsFormula(formula, allowedSeries)), - { concurrency: 4 }, - ); - const definitionsBySpecificity = [...definition.series].sort( - (left, right) => right.key.length - left.key.length, - ); - const groups = new Map(); - - for (const source of sourceSeries) { - const comparison = source.comparison ?? "current"; - const sourceKey = stripSuffix(source.key, trendsComparisonKeySuffix(comparison)); - const definitionSeries = definitionsBySpecificity.find( - (candidate) => sourceKey === candidate.key || sourceKey.startsWith(`${candidate.key}:`), - ); - if (!definitionSeries) continue; - const breakdownValue = breakdownValueFromKey(sourceKey, definitionSeries.key); - const groupKey = `${comparison}\u0000${breakdownValue ?? ""}`; - const breakdownFields: { breakdownValue?: string } = {}; - if (breakdownValue !== undefined) breakdownFields.breakdownValue = breakdownValue; - const group = groups.get(groupKey) ?? { - ...breakdownFields, - comparison, - series: new Map>(), - }; - group.series.set( - definitionSeries.key.toLowerCase(), - new Map(source.points.map((point) => [point.timestamp.getTime(), point.value])), - ); - groups.set(groupKey, group); - } - - const result: Array = []; - for (const group of groups.values()) { - const timestamps = new Set(); - for (const points of group.series.values()) { - for (const timestamp of points.keys()) timestamps.add(timestamp); - } - const sortedTimestamps = [...timestamps].sort((left, right) => left - right); - for (const [index, formula] of definition.formulas.entries()) { - const node = compiled[index]; - if (!node) continue; - const formulaLabel = formula.label ?? `Formula (${formula.expression})`; - result.push({ - comparison: group.comparison, - key: `${keyWithBreakdown(formula.key, group.breakdownValue)}${trendsComparisonKeySuffix(group.comparison)}`, - label: labelWithComparison( - labelWithBreakdown(formulaLabel, group.breakdownValue), - group.comparison, - ), - points: sortedTimestamps.map((timestamp) => { - const values = new Map(); - for (const seriesKey of allowedSeries) { - values.set(seriesKey, group.series.get(seriesKey)?.get(timestamp) ?? 0); - } - return { - timestamp: dateFrom(timestamp), - value: evaluateTrendsFormulaNode(node, values), - }; - }), - }); - } - } - return result; - }); - -/** Validate a funnel definition before lowering it to ClickHouse. */ -export const validateExecutableFunnelsDefinition = ( - definition: CustomAnalyticsInsightQueryType, -): Effect.Effect => - Effect.gen(function* () { - if (definition.kind !== "funnels") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${definition.kind} is not a funnel definition`, - }), - ); - } - if (definition.steps.length < 2 || definition.steps.length > 20) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Funnels require between 2 and 20 steps" }), - ); - } - if ( - !Number.isSafeInteger(definition.conversionWindowSeconds) || - definition.conversionWindowSeconds < 1 || - definition.conversionWindowSeconds > 31_536_000 - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Funnel conversion windows must be whole seconds between 1 second and 365 days", - }), - ); - } - if (definition.breakdown) { - yield* validateCustomEventField(definition.breakdown.field); - const attributionStep = definition.breakdownAttributionStep ?? 1; - if (!Number.isSafeInteger(attributionStep) || attributionStep > definition.steps.length) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Funnel breakdown attribution must reference an existing step", - }), - ); - } - } else if (definition.breakdownAttributionStep !== undefined) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Funnel breakdown attribution requires a breakdown field", - }), - ); - } - let predicateCount = 0; - for (const step of definition.steps) { - if (step.filters) predicateCount += yield* validateCustomEventFilter(step.filters); - } - if (predicateCount > 20) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Funnels support at most 20 predicates" }), - ); - } - return definition; - }); - -type FunnelsInsightResult = Extract< - QueryCustomAnalyticsInsightResponseType, - { readonly kind: "funnels" } ->; - -/** Convert monotonic funnel reach counts into display-ready step metrics. */ -export const buildFunnelStepResults = ( - definition: ExecutableFunnelsDefinition, - counts: ReadonlyArray, -): FunnelsInsightResult["steps"] => { - const entryCount = counts[0] ?? 0; - return definition.steps.map((step, index) => { - const count = counts[index] ?? 0; - const previousCount = counts[index - 1] ?? count; - const dropoffCount = Math.max(0, previousCount - count); - return { - conversionRate: safeRatio(count, entryCount), - count, - dropoffCount, - dropoffRate: safeRatio(dropoffCount, previousCount), - key: step.key, - label: step.label ?? step.eventNames.join(" or "), - step: index + 1, - }; - }); -}; - -/** Overall conversion from the funnel entry step to its final step. */ -const funnelTotalConversionRate = ( - steps: FunnelsInsightResult["steps"], - entryCount: number, -): number => safeRatio(steps.at(-1)?.count ?? 0, entryCount); - -/** Validate a retention definition before lowering it to ClickHouse. */ -export const validateExecutableRetentionDefinition = ( - definition: CustomAnalyticsInsightQueryType, -): Effect.Effect => - Effect.gen(function* () { - if (definition.kind !== "retention") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${definition.kind} is not a retention definition`, - }), - ); - } - const intervals = definition.intervals ?? 11; - if (!Number.isSafeInteger(intervals) || intervals < 1 || intervals > 24) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Retention supports between 1 and 24 intervals", - }), - ); - } - let predicateCount = 0; - if (definition.start.filters) { - predicateCount += yield* validateCustomEventFilter(definition.start.filters); - } - if (definition.returning.filters) { - predicateCount += yield* validateCustomEventFilter(definition.returning.filters); - } - if (predicateCount > 20) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Retention supports at most 20 predicates" }), - ); - } - return definition; - }); - -type RetentionInsightResult = Extract< - QueryCustomAnalyticsInsightResponseType, - { readonly kind: "retention" } ->; - -const retentionDenominator = ( - definition: ExecutableRetentionDefinition, - cohort: EventRetentionCohort, - interval: number, -): number => { - if (definition.reference === "previous" && interval > 0) return cohort.counts[interval - 1] ?? 0; - return cohort.cohortSize; -}; - -/** Convert retention counts into cohort- or previous-period-relative cells. */ -export const buildRetentionCohortResults = ( - definition: ExecutableRetentionDefinition, - cohorts: ReadonlyArray, -): RetentionInsightResult["cohorts"] => - cohorts.map((cohort) => ({ - cells: cohort.counts.map((count, interval) => { - const denominator = retentionDenominator(definition, cohort, interval); - return { - count, - interval, - rate: safeRatio(count, denominator), - }; - }), - cohortSize: cohort.cohortSize, - cohortStart: cohort.cohortStart, - })); - -/** Validate a paths definition before lowering it to ClickHouse. */ -export const validateExecutablePathsDefinition = ( - definition: CustomAnalyticsInsightQueryType, -): Effect.Effect => - Effect.gen(function* () { - if (definition.kind !== "paths") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${definition.kind} is not a paths definition`, - }), - ); - } - if ( - !Number.isSafeInteger(definition.maxDepth) || - definition.maxDepth < 2 || - definition.maxDepth > 20 - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Paths require between 2 and 20 steps" }), - ); - } - if (definition.startEventName !== undefined && definition.startEventName.trim().length === 0) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Paths start event cannot be empty", - }), - ); - } - if (definition.endEventName !== undefined && definition.endEventName.trim().length === 0) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Paths end event cannot be empty", - }), - ); - } - const sessionGapSeconds = definition.sessionGapSeconds ?? 1_800; - if ( - !Number.isSafeInteger(sessionGapSeconds) || - sessionGapSeconds < 60 || - sessionGapSeconds > 86_400 - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Path session gaps must be whole seconds between 1 minute and 24 hours", - }), - ); - } - const edgeLimit = definition.edgeLimit ?? 50; - if (!Number.isSafeInteger(edgeLimit) || edgeLimit < 1 || edgeLimit > 200) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Paths support between 1 and 200 links" }), - ); - } - if ( - definition.minEdgeCount !== undefined && - definition.maxEdgeCount !== undefined && - definition.minEdgeCount > definition.maxEdgeCount - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Path minimum link count cannot exceed the maximum", - }), - ); - } - if (definition.eventNames.length > 200 || (definition.excludeEventNames?.length ?? 0) > 200) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Paths support at most 200 event selectors" }), - ); - } - if (definition.filters && (yield* validateCustomEventFilter(definition.filters)) > 20) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Paths support at most 20 predicates" }), - ); - } - return definition; - }); - -type PathsInsightResult = Extract< - QueryCustomAnalyticsInsightResponseType, - { readonly kind: "paths" } ->; - -/** Normalize ClickHouse path links into the public insight result. */ -export const buildPathsLinkResults = ( - links: ReadonlyArray, -): PathsInsightResult["links"] => - links.map((link) => ({ - averageTransitionSeconds: link.averageTransitionSeconds, - count: link.count, - source: link.source, - sourceStep: link.sourceStep, - target: link.target, - targetStep: link.targetStep, - })); - -/** Validate a stickiness definition before lowering it to ClickHouse. */ -export const validateExecutableStickinessDefinition = ( - definition: CustomAnalyticsInsightQueryType, -): Effect.Effect => - Effect.gen(function* () { - if (definition.kind !== "stickiness") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${definition.kind} is not a stickiness definition`, - }), - ); - } - if (definition.series.length > 8) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Stickiness supports at most 8 series" }), - ); - } - const occurrenceCriteria = definition.occurrenceCriteria ?? { operator: "gte", value: 1 }; - if ( - !Number.isSafeInteger(occurrenceCriteria.value) || - occurrenceCriteria.value < 1 || - occurrenceCriteria.value > 10_000 - ) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Stickiness minimum occurrences must be a whole number between 1 and 10,000", - }), - ); - } - let predicateCount = 0; - for (const series of definition.series) { - if (series.aggregation !== "unique_users") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Stickiness series must use unique users", - }), - ); - } - if (series.filters) predicateCount += yield* validateCustomEventFilter(series.filters); - } - if (predicateCount > 20) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Stickiness supports at most 20 predicates" }), - ); - } - return definition; - }); - -const startOfStickinessInterval = ( - date: Date, - interval: ExecutableStickinessDefinition["interval"], -): Date => { - const value = dateFrom(date); - value.setUTCMinutes(0, 0, 0); - if (interval === "hour") return value; - value.setUTCHours(0); - if (interval === "day") return value; - if (interval === "week") { - value.setUTCDate(value.getUTCDate() - ((value.getUTCDay() + 6) % 7)); - return value; - } - value.setUTCDate(1); - return value; -}; - -const stickinessIntervalMillis = ( - interval: ExecutableStickinessDefinition["interval"], -): number => { - if (interval === "hour") return 3_600_000; - if (interval === "week") return 604_800_000; - return 86_400_000; -}; - -/** Count inclusive activity intervals represented by a resolved time range. */ -export const countStickinessIntervals = ( - start: Date, - end: Date, - interval: ExecutableStickinessDefinition["interval"], -): number => { - const from = startOfStickinessInterval(start, interval); - const to = startOfStickinessInterval(end, interval); - if (interval === "month") { - return ( - (to.getUTCFullYear() - from.getUTCFullYear()) * 12 + to.getUTCMonth() - from.getUTCMonth() + 1 - ); - } - return Math.floor((to.getTime() - from.getTime()) / stickinessIntervalMillis(interval)) + 1; -}; - -const stickinessBucketCount = ( - raw: ReadonlyArray, - counts: ReadonlyMap, - computation: "cumulative" | "exact", - intervals: number, -): number => { - if (computation !== "cumulative") return counts.get(intervals) ?? 0; - return raw.reduce((total, bucket) => { - if (bucket.intervals >= intervals) return total + bucket.count; - return total; - }, 0); -}; - -/** Fill sparse frequency counts and optionally convert them to at-least-N counts. */ -export const buildStickinessBuckets = ( - raw: ReadonlyArray, - computation: "cumulative" | "exact", - maximumIntervals: number, -): EventStickinessBucket[] => { - const counts = new Map(raw.map((bucket) => [bucket.intervals, bucket.count])); - return Array.from({ length: maximumIntervals }, (_, offset) => { - const intervals = offset + 1; - return { - count: stickinessBucketCount(raw, counts, computation, intervals), - intervals, - }; - }); -}; - -/** Validate a lifecycle definition before lowering it to ClickHouse. */ -export const validateExecutableLifecycleDefinition = ( - definition: CustomAnalyticsInsightQueryType, -): Effect.Effect => - Effect.gen(function* () { - if (definition.kind !== "lifecycle") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: `${definition.kind} is not a lifecycle definition`, - }), - ); - } - if (definition.series.aggregation !== "unique_users") { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Lifecycle must use unique users" }), - ); - } - if (definition.series.filters) { - const predicateCount = yield* validateCustomEventFilter(definition.series.filters); - if (predicateCount > 20) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Lifecycle supports at most 20 predicates" }), - ); - } - } - if (definition.statuses && new Set(definition.statuses).size !== definition.statuses.length) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Lifecycle statuses cannot be duplicated" }), - ); - } - return definition; - }); - -type LifecycleInsightResult = Extract< - QueryCustomAnalyticsInsightResponseType, - { readonly kind: "lifecycle" } ->; - -const nextLifecycleInterval = ( - value: Date, - granularity: ExecutableLifecycleDefinition["granularity"], -): Date => { - const next = dateFrom(value); - if (granularity === "hour") next.setUTCHours(next.getUTCHours() + 1); - else if (granularity === "day") next.setUTCDate(next.getUTCDate() + 1); - else if (granularity === "week") next.setUTCDate(next.getUTCDate() + 7); - else next.setUTCMonth(next.getUTCMonth() + 1); - return next; -}; - -/** Fill every selected lifecycle status across the inclusive resolved time range. */ -export const buildLifecycleSeries = ( - points: ReadonlyArray, - statuses: ReadonlyArray, - start: Date, - end: Date, - granularity: ExecutableLifecycleDefinition["granularity"], -): LifecycleInsightResult["series"] => { - const from = startOfStickinessInterval(start, granularity); - const to = startOfStickinessInterval(end, granularity); - const periods: Date[] = []; - for (let period = from; period <= to; period = nextLifecycleInterval(period, granularity)) { - periods.push(period); - } - const counts = new Map( - points.map((point) => [`${point.status}:${point.timestamp.getTime()}`, point.count]), - ); - return statuses.map((status) => ({ - points: periods.map((timestamp) => ({ - count: counts.get(`${status}:${timestamp.getTime()}`) ?? 0, - timestamp, - })), - status, - })); -}; - -/** Saved custom insight and dashboard authoring plus executable insight queries. */ -export class CustomAnalyticsService extends Context.Service()( - "CustomAnalyticsService", - { - make: Effect.gen(function* () { - const ch = Option.getOrUndefined( - yield* Effect.serviceOption(ClickhouseWebClient.ClickhouseWebClient), - ); - const db = yield* Db; - - /** - * Run a ClickHouse-backed insight query, or yield no rows at all when the - * deployment has no analytics storage bound. - */ - const queryClickhouseRows = ( - effect: Effect.Effect, E, ClickhouseWebClient.ClickhouseWebClient>, - ): Effect.Effect, E> => { - if (ch === undefined) return Effect.succeed([]); - return effect.pipe(Effect.provideService(ClickhouseWebClient.ClickhouseWebClient, ch)); - }; - - const getProject = Effect.fn("customAnalytics.getProject")(function* (projectId: string) { - yield* checkProjectPermission( - projectId, - "project:all", - `Not authorized to access analytics for project ${projectId}`, - ); - const project = yield* db.query.projects.findFirst({ - where: { id: projectId }, - }); - if (!project) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: projectId, - message: "Analytics project not found", - }), - ); - } - return project; - }); - - const loadCohortRow = Effect.fn("customAnalytics.loadCohort")(function* (id: string) { - const [cohort] = yield* db - .select() - .from(analyticsCohorts) - .where(and(eq(analyticsCohorts.id, id), isNull(analyticsCohorts.deletedAt))) - .limit(1); - if (!cohort) { - return yield* Effect.fail( - new AnalyticsServiceError({ cause: id, message: "Analytics cohort not found" }), - ); - } - yield* getProject(cohort.projectId); - return cohort; - }); - - const hydrateCohort = Effect.fn("customAnalytics.hydrateCohort")(function* ( - cohort: AnalyticsCohortRow, - ) { - const members = yield* db - .select({ personId: analyticsCohortMembers.personId }) - .from(analyticsCohortMembers) - .where(eq(analyticsCohortMembers.cohortId, cohort.id)); - return { - createdAt: cohort.createdAt, - createdBy: cohort.createdBy, - description: cohort.description, - id: cohort.id, - memberCount: members.length, - memberPersonIds: members.map((member) => member.personId), - name: cohort.name, - organizationId: cohort.organizationId, - projectId: cohort.projectId, - updatedAt: cohort.updatedAt, - } satisfies AnalyticsCohortType; - }); - - const validateCohortMembers = Effect.fn("customAnalytics.validateCohortMembers")(function* ( - projectId: string, - personIds: ReadonlyArray, - ) { - const normalized = [...new Set(personIds.map((id) => id.trim()).filter(Boolean))]; - if (normalized.length === 0) return normalized; - const matching = yield* db - .select({ id: persons.id }) - .from(persons) - .where( - and( - eq(persons.projectId, projectId), - inArray(persons.id, normalized), - isNull(persons.archivedAt), - isNull(persons.deletedAt), - isNull(persons.mergedIntoPersonId), - ), - ); - if (matching.length !== normalized.length) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: projectId, - message: "Every cohort member must be an active person in the cohort project", - }), - ); - } - return normalized; - }); - - const resolveCohortPersonIds = Effect.fn("customAnalytics.resolveCohortPersonIds")(function* ( - projectId: string, - cohortIds: ReadonlyArray | undefined, - ) { - if (cohortIds === undefined) return undefined; - const uniqueCohortIds = [...new Set(cohortIds)]; - if (uniqueCohortIds.length === 0) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ message: "Cohort filters cannot be empty" }), - ); - } - for (const cohortId of uniqueCohortIds) { - const cohort = yield* loadCohortRow(cohortId); - if (cohort.projectId !== projectId) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: cohortId, - message: "Analytics cohort belongs to a different project", - }), - ); - } - } - const members = yield* db - .select({ personId: analyticsCohortMembers.personId }) - .from(analyticsCohortMembers) - .where(inArray(analyticsCohortMembers.cohortId, uniqueCohortIds)); - return [...new Set(members.map((member) => member.personId))]; - }); - - const loadInsightRow = Effect.fn("customAnalytics.loadInsight")(function* (id: string) { - const [insight] = yield* db - .select() - .from(analyticsInsights) - .where(and(eq(analyticsInsights.id, id), isNull(analyticsInsights.deletedAt))) - .limit(1); - if (!insight) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: id, - message: "Analytics insight not found", - }), - ); - } - yield* getProject(insight.projectId); - return insight; - }); - - const hydrateDashboard = Effect.fn("customAnalytics.hydrateDashboard")(function* ( - dashboard: AnalyticsDashboardRow, - ) { - const itemRows = yield* db - .select() - .from(analyticsDashboardItems) - .where(eq(analyticsDashboardItems.dashboardId, dashboard.id)) - .orderBy(asc(analyticsDashboardItems.position)); - const items: Array = []; - for (const item of itemRows) { - if (item.sourceType === "insight") { - const [insight] = yield* db - .select() - .from(analyticsInsights) - .where( - and( - eq(analyticsInsights.id, item.sourceId), - eq(analyticsInsights.projectId, dashboard.projectId), - isNull(analyticsInsights.deletedAt), - ), - ) - .limit(1); - if (insight) { - items.push({ - id: item.id, - insight: yield* toSavedInsight(insight), - kind: "insight", - layout: item.layout, - position: item.position, - }); - } - } else if (item.sourceType === "voidql") { - const [query] = yield* db - .select() - .from(analyticsSavedQuery) - .where( - and( - eq(analyticsSavedQuery.id, item.sourceId), - eq(analyticsSavedQuery.organizationId, dashboard.organizationId), - ), - ) - .limit(1); - if (query) { - items.push({ - id: item.id, - kind: "voidql", - layout: item.layout, - position: item.position, - query: toSavedVoidQlInsight(query), - }); - } - } - } - return { - createdAt: dashboard.createdAt, - createdBy: dashboard.createdBy, - description: dashboard.description, - id: dashboard.id, - items, - name: dashboard.name, - organizationId: dashboard.organizationId, - projectId: dashboard.projectId, - updatedAt: dashboard.updatedAt, - } satisfies AnalyticsDashboardType; - }); - - const loadDashboardRow = Effect.fn("customAnalytics.loadDashboard")(function* (id: string) { - const [dashboard] = yield* db - .select() - .from(analyticsDashboards) - .where(and(eq(analyticsDashboards.id, id), isNull(analyticsDashboards.deletedAt))) - .limit(1); - if (!dashboard) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: id, - message: "Analytics dashboard not found", - }), - ); - } - yield* getProject(dashboard.projectId); - return dashboard; - }); - - /** Execute a supported custom insight definition. */ - const queryInsight = Effect.fn("customAnalytics.queryInsight")( - function* (input: { - readonly definition: CustomAnalyticsInsightQueryType; - readonly projectId: string; - }) { - const project = yield* getProject(input.projectId); - if (input.definition.actor?.kind === "group") { - yield* validateCustomEventField( - `${CUSTOM_PROPERTY_PREFIX}${input.definition.actor.property}`, - ); - } - const cohortPersonIds = yield* resolveCohortPersonIds( - input.projectId, - input.definition.cohortIds, - ); - const actorScope: { - actor?: AnalyticsActorType; - cohortPersonIds?: ReadonlyArray; - } = {}; - if (input.definition.actor) actorScope.actor = input.definition.actor; - if (cohortPersonIds !== undefined) actorScope.cohortPersonIds = cohortPersonIds; - if (input.definition.kind === "trends") { - const definition = yield* validateExecutableTrendsDefinition(input.definition); - const resolvedTimeRange = yield* resolveTimeRange(definition.timeRange); - const comparisonTimeRange = resolveOptionalTrendsComparisonTimeRange( - definition.comparison, - resolvedTimeRange, - ); - const alignedPoints = ( - points: TrendsInsightResult["series"][number]["points"], - comparison: "current" | AnalyticsTrendsComparisonType, - ): TrendsInsightResult["series"][number]["points"] => { - if (comparisonTimeRange === undefined || comparison === "current") return points; - return alignTrendsComparisonPoints( - points, - resolvedTimeRange, - comparisonTimeRange, - definition.granularity, - ); - }; - const queryPeriod = ( - range: ResolvedTrendsTimeRange, - comparison: "current" | AnalyticsTrendsComparisonType, - ) => - Effect.all( - definition.series.map((seriesDefinition) => - Effect.gen(function* () { - const groups = yield* queryClickhouseRows( - getEventTrendSeries({ - ...actorScope, - aggregation: seriesDefinition.aggregation, - aggregateOverRange: definition.display === "number", - breakdown: definition.breakdown, - eventNames: seriesDefinition.eventNames, - filters: { projectIds: [input.projectId] }, - mathProperty: seriesDefinition.mathProperty, - organizationId: project.organizationId, - params: { - endDate: range.end, - granularity: definition.granularity, - startDate: range.start, - }, - propertyFilter: seriesDefinition.filters, - }), - ); - const baseLabel = - seriesDefinition.label ?? seriesDefinition.eventNames.join(" or "); - const effectiveGroups = [...groups]; - if (effectiveGroups.length === 0 && !definition.breakdown) { - effectiveGroups.push({ points: [] }); - } - return effectiveGroups.map((group) => ({ - comparison, - key: `${keyWithBreakdown(seriesDefinition.key, group.breakdownValue)}${trendsComparisonKeySuffix(comparison)}`, - label: labelWithComparison( - labelWithBreakdown(baseLabel, group.breakdownValue), - comparison, - ), - points: alignedPoints(group.points, comparison), - })); - }), - ), - { concurrency: 4 }, - ); - const periodEffects = [queryPeriod(resolvedTimeRange, "current")]; - if (comparisonTimeRange !== undefined && definition.comparison !== undefined) { - periodEffects.push(queryPeriod(comparisonTimeRange, definition.comparison)); - } - const periodSeries = yield* Effect.all(periodEffects, { concurrency: 2 }); - const presentPoints = ( - points: TrendsInsightResult["series"][number]["points"], - ): TrendsInsightResult["series"][number]["points"] => { - if (definition.display === "number") return points; - return fillTrendsSeriesPoints(points, resolvedTimeRange, definition.granularity); - }; - const queriedSeries = periodSeries.flat(2).map((series) => ({ - ...series, - points: presentPoints(series.points), - })); - const resultSeries: TrendsInsightResult["series"] = yield* Effect.gen(function* () { - if (!definition.formulas?.length) return queriedSeries; - return yield* buildTrendsFormulaSeries(definition, queriedSeries); - }); - const presentedSeries = resultSeries.map((series) => ({ - ...series, - points: applyTrendsPresentation(series.points, definition), - })); - const comparisonFields: { comparisonTimeRange?: ResolvedTrendsTimeRange } = {}; - if (comparisonTimeRange !== undefined) { - comparisonFields.comparisonTimeRange = comparisonTimeRange; - } - return { - ...comparisonFields, - kind: constant("trends"), - resolvedTimeRange, - series: presentedSeries, - }; - } - if (input.definition.kind === "funnels") { - const definition = yield* validateExecutableFunnelsDefinition(input.definition); - const resolvedTimeRange = yield* resolveTimeRange(definition.timeRange); - const breakdownFields: { - breakdown?: ExecutableFunnelsDefinition["breakdown"]; - breakdownAttributionStep?: number; - } = {}; - if (definition.breakdown) { - breakdownFields.breakdown = definition.breakdown; - breakdownFields.breakdownAttributionStep = definition.breakdownAttributionStep ?? 1; - } - const funnelInput = { - ...actorScope, - ...breakdownFields, - conversionWindowSeconds: definition.conversionWindowSeconds, - filters: { projectIds: [input.projectId] }, - order: definition.order, - organizationId: project.organizationId, - params: { - endDate: resolvedTimeRange.end, - startDate: resolvedTimeRange.start, - }, - steps: definition.steps, - }; - const [counts, breakdownCounts] = yield* Effect.all( - [ - queryClickhouseRows(getEventFunnelCounts(funnelInput)), - queryClickhouseRows(getEventFunnelBreakdownCounts(funnelInput)), - ], - { concurrency: 2 }, - ); - const steps = buildFunnelStepResults(definition, counts); - const entryCount = steps[0]?.count ?? 0; - const breakdownResults: { - breakdowns?: ReadonlyArray<{ - readonly breakdownValue: string; - readonly steps: FunnelsInsightResult["steps"]; - readonly totalConversionRate: number; - }>; - } = {}; - if (definition.breakdown) { - breakdownResults.breakdowns = breakdownCounts.map((group) => { - const groupSteps = buildFunnelStepResults(definition, group.counts); - return { - breakdownValue: group.breakdownValue, - steps: groupSteps, - totalConversionRate: funnelTotalConversionRate( - groupSteps, - groupSteps[0]?.count ?? 0, - ), - }; - }); - } - return { - ...breakdownResults, - kind: constant("funnels"), - resolvedTimeRange, - steps, - totalConversionRate: funnelTotalConversionRate(steps, entryCount), - }; - } - if (input.definition.kind === "retention") { - const definition = yield* validateExecutableRetentionDefinition(input.definition); - const resolvedTimeRange = yield* resolveTimeRange(definition.timeRange); - const cohorts = yield* queryClickhouseRows( - getEventRetentionCohorts({ - ...actorScope, - cumulative: definition.cumulative ?? false, - filters: { projectIds: [input.projectId] }, - intervals: definition.intervals ?? 11, - organizationId: project.organizationId, - params: { - endDate: resolvedTimeRange.end, - startDate: resolvedTimeRange.start, - }, - period: definition.period, - retentionType: definition.retentionType ?? "recurring", - returning: definition.returning, - start: definition.start, - }), - ); - return { - cohorts: buildRetentionCohortResults(definition, cohorts), - kind: constant("retention"), - period: definition.period, - resolvedTimeRange, - }; - } - if (input.definition.kind === "paths") { - const definition = yield* validateExecutablePathsDefinition(input.definition); - const resolvedTimeRange = yield* resolveTimeRange(definition.timeRange); - const sessionGapSeconds = definition.sessionGapSeconds ?? 1_800; - const links = yield* queryClickhouseRows( - getEventPathLinks({ - ...actorScope, - definition, - filters: { projectIds: [input.projectId] }, - organizationId: project.organizationId, - params: { - endDate: resolvedTimeRange.end, - startDate: resolvedTimeRange.start, - }, - }), - ); - return { - kind: constant("paths"), - links: buildPathsLinkResults(links), - maxDepth: definition.maxDepth, - resolvedTimeRange, - sessionGapSeconds, - }; - } - if (input.definition.kind === "stickiness") { - const definition = yield* validateExecutableStickinessDefinition(input.definition); - const resolvedTimeRange = yield* resolveTimeRange(definition.timeRange); - const computation = definition.computation ?? "exact"; - const occurrenceCriteria = - definition.occurrenceCriteria ?? constant({ operator: "gte", value: 1 }); - const maximumIntervals = countStickinessIntervals( - resolvedTimeRange.start, - resolvedTimeRange.end, - definition.interval, - ); - if (maximumIntervals > 10_000) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Stickiness time ranges support at most 10,000 intervals", - }), - ); - } - const series = yield* Effect.all( - definition.series.map((seriesDefinition) => - Effect.gen(function* () { - const raw = yield* queryClickhouseRows( - getEventStickinessBuckets({ - ...actorScope, - filters: { projectIds: [input.projectId] }, - interval: definition.interval, - occurrenceCriteria, - organizationId: project.organizationId, - params: { - endDate: resolvedTimeRange.end, - startDate: resolvedTimeRange.start, - }, - series: seriesDefinition, - }), - ); - return { - buckets: buildStickinessBuckets(raw, computation, maximumIntervals), - key: seriesDefinition.key, - label: seriesDefinition.label ?? seriesDefinition.eventNames.join(" or "), - }; - }), - ), - { concurrency: 4 }, - ); - return { - computation, - interval: definition.interval, - kind: constant("stickiness"), - resolvedTimeRange, - series, - }; - } - if (input.definition.kind === "lifecycle") { - const definition = yield* validateExecutableLifecycleDefinition(input.definition); - const resolvedTimeRange = yield* resolveTimeRange(definition.timeRange); - const points = yield* queryClickhouseRows( - getEventLifecyclePoints({ - ...actorScope, - filters: { projectIds: [input.projectId] }, - granularity: definition.granularity, - organizationId: project.organizationId, - params: { - endDate: resolvedTimeRange.end, - startDate: resolvedTimeRange.start, - }, - series: definition.series, - }), - ); - const statuses = - definition.statuses ?? - constant(["new", "returning", "resurrecting", "dormant"]); - return { - granularity: definition.granularity, - kind: constant("lifecycle"), - resolvedTimeRange, - series: buildLifecycleSeries( - points, - statuses, - resolvedTimeRange.start, - resolvedTimeRange.end, - definition.granularity, - ), - }; - } - return yield* Effect.die("Unhandled custom analytics insight kind"); - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to query the custom analytics insight", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to query the custom analytics insight", - }), - ), - }), - ), - ); - - /** List people behind a selected custom-insight event segment. */ - const queryPersons = Effect.fn("customAnalytics.queryPersons")( - function* (input: { - readonly cohortIds?: ReadonlyArray; - readonly eventNames: ReadonlyArray; - readonly filters?: AnalyticsFilterType; - readonly group?: { readonly property: string; readonly value: string }; - readonly limit?: number; - readonly projectId: string; - readonly timeRange: CustomAnalyticsInsightQueryType["timeRange"]; - }) { - const project = yield* getProject(input.projectId); - const eventNames = [ - ...new Set(input.eventNames.map((name) => name.trim()).filter(Boolean)), - ]; - if (eventNames.length === 0) { - return yield* Effect.fail( - new InvalidAnalyticsQueryError({ - message: "Person drilldowns require at least one event name", - }), - ); - } - if (input.filters) yield* validateCustomEventFilter(input.filters); - if (input.group) { - yield* validateCustomEventField(`${CUSTOM_PROPERTY_PREFIX}${input.group.property}`); - } - const resolvedTimeRange = yield* resolveTimeRange(input.timeRange); - const cohortPersonIds = yield* resolveCohortPersonIds(input.projectId, input.cohortIds); - const drilldownScope: { - cohortPersonIds?: ReadonlyArray; - filters?: AnalyticsFilterType; - group?: { readonly property: string; readonly value: string }; - } = {}; - if (cohortPersonIds !== undefined) drilldownScope.cohortPersonIds = cohortPersonIds; - if (input.filters) drilldownScope.filters = input.filters; - if (input.group) drilldownScope.group = input.group; - const rows = yield* queryClickhouseRows( - getEventPersonDrilldown({ - ...drilldownScope, - eventNames, - limit: Math.min(Math.max(input.limit ?? 50, 1), 100), - organizationId: project.organizationId, - params: { - endDate: resolvedTimeRange.end, - startDate: resolvedTimeRange.start, - }, - projectId: project.id, - }), - ); - if (rows.length === 0) return { people: [], resolvedTimeRange }; - const personRows = yield* db - .select({ email: persons.email, id: persons.id, name: persons.name }) - .from(persons) - .where( - and( - eq(persons.projectId, project.id), - inArray( - persons.id, - rows.map((row) => row.personId), - ), - isNull(persons.archivedAt), - isNull(persons.deletedAt), - isNull(persons.mergedIntoPersonId), - ), - ); - const personById = new Map(personRows.map((person) => [person.id, person])); - return { - people: rows.flatMap((row) => { - const person = personById.get(row.personId); - if (!person) return []; - return [ - { - email: person.email, - eventCount: row.eventCount, - lastSeenAt: row.lastSeenAt, - name: person.name, - personId: row.personId, - }, - ]; - }), - resolvedTimeRange, - }; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to query custom analytics people", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to query custom analytics people", - }), - ), - }), - ), - ); - - /** List saved insights for a project. */ - const listInsights = Effect.fn("customAnalytics.listInsights")( - function* (input: { readonly projectId: string }) { - yield* getProject(input.projectId); - const rows = yield* db - .select() - .from(analyticsInsights) - .where( - and( - eq(analyticsInsights.projectId, input.projectId), - isNull(analyticsInsights.deletedAt), - ), - ) - .orderBy(desc(analyticsInsights.updatedAt)); - return { insights: yield* Effect.forEach(rows, toSavedInsight) }; - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to list analytics insights", - }), - ), - ), - ), - ); - - /** Persist a new project insight definition. */ - const createInsight = Effect.fn("customAnalytics.createInsight")( - function* (input: { - readonly definition: CustomAnalyticsInsightQueryType; - readonly description?: string; - readonly name: string; - readonly projectId: string; - }) { - const project = yield* getProject(input.projectId); - const session = yield* AuthSession; - const id = generateId("analyticsInsight"); - yield* db.insert(analyticsInsights).values({ - createdBy: session?.user?.id ?? "system", - definition: input.definition, - description: input.description, - id, - kind: input.definition.kind, - name: input.name, - organizationId: project.organizationId, - projectId: project.id, - }); - return yield* toSavedInsight(yield* loadInsightRow(id)); - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to create analytics insight", - }), - ), - ), - ), - ); - - /** Update a saved insight's metadata or definition. */ - const updateInsight = Effect.fn("customAnalytics.updateInsight")( - function* (input: { - readonly definition?: CustomAnalyticsInsightQueryType; - readonly description?: string | null; - readonly id: string; - readonly name?: string; - }) { - yield* loadInsightRow(input.id); - const changes: { - definition?: CustomAnalyticsInsightQueryType; - description?: string | null; - kind?: CustomAnalyticsInsightQueryType["kind"]; - name?: string; - } = {}; - if (input.definition !== undefined) { - changes.definition = input.definition; - changes.kind = input.definition.kind; - } - if (input.description !== undefined) changes.description = input.description; - if (input.name !== undefined) changes.name = input.name; - yield* db - .update(analyticsInsights) - .set({ - ...changes, - updatedAt: currentTimestamp(), - }) - .where(eq(analyticsInsights.id, input.id)); - return yield* toSavedInsight(yield* loadInsightRow(input.id)); - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to update analytics insight", - }), - ), - ), - ), - ); - - /** Soft-delete a saved insight and remove it from dashboards. */ - const deleteInsight = Effect.fn("customAnalytics.deleteInsight")( - function* (input: { readonly id: string }) { - yield* loadInsightRow(input.id); - yield* db.transaction((tx) => - Effect.gen(function* () { - yield* tx - .update(analyticsInsights) - .set({ deletedAt: currentTimestamp(), updatedAt: currentTimestamp() }) - .where(eq(analyticsInsights.id, input.id)); - yield* tx - .delete(analyticsDashboardItems) - .where( - and( - eq(analyticsDashboardItems.sourceType, "insight"), - eq(analyticsDashboardItems.sourceId, input.id), - ), - ); - }), - ); - return { deleted: true }; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to delete analytics insight", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to delete analytics insight", - }), - ), - }), - ), - ); - - /** List reusable static cohorts for a project. */ - const listCohorts = Effect.fn("customAnalytics.listCohorts")( - function* (input: { readonly projectId: string }) { - yield* getProject(input.projectId); - const rows = yield* db - .select() - .from(analyticsCohorts) - .where( - and( - eq(analyticsCohorts.projectId, input.projectId), - isNull(analyticsCohorts.deletedAt), - ), - ) - .orderBy(desc(analyticsCohorts.updatedAt)); - return { cohorts: yield* Effect.forEach(rows, hydrateCohort) }; - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to list analytics cohorts", - }), - ), - ), - ), - ); - - /** Create a reusable static cohort from canonical project people. */ - const createCohort = Effect.fn("customAnalytics.createCohort")( - function* (input: { - readonly description?: string; - readonly memberPersonIds: ReadonlyArray; - readonly name: string; - readonly projectId: string; - }) { - const project = yield* getProject(input.projectId); - const session = yield* AuthSession; - const memberPersonIds = yield* validateCohortMembers(project.id, input.memberPersonIds); - const id = generateId("analyticsCohort"); - yield* db.transaction((tx) => - Effect.gen(function* () { - yield* tx.insert(analyticsCohorts).values({ - createdBy: session?.user?.id ?? "system", - description: input.description, - id, - name: input.name, - organizationId: project.organizationId, - projectId: project.id, - }); - if (memberPersonIds.length > 0) { - yield* tx.insert(analyticsCohortMembers).values( - memberPersonIds.map((personId) => ({ - cohortId: id, - id: generateId("analyticsCohortMember"), - personId, - })), - ); - } - }), - ); - return yield* hydrateCohort(yield* loadCohortRow(id)); - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to create analytics cohort", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to create analytics cohort", - }), - ), - }), - ), - ); - - /** Update cohort metadata or atomically replace its static membership. */ - const updateCohort = Effect.fn("customAnalytics.updateCohort")( - function* (input: { - readonly description?: string | null; - readonly id: string; - readonly memberPersonIds?: ReadonlyArray; - readonly name?: string; - }) { - const cohort = yield* loadCohortRow(input.id); - const memberPersonIds = yield* Effect.gen(function* () { - if (input.memberPersonIds === undefined) return undefined; - return yield* validateCohortMembers(cohort.projectId, input.memberPersonIds); - }); - const changes: { description?: string | null; name?: string } = {}; - if (input.description !== undefined) changes.description = input.description; - if (input.name !== undefined) changes.name = input.name; - yield* db.transaction((tx) => - Effect.gen(function* () { - yield* tx - .update(analyticsCohorts) - .set({ - ...changes, - updatedAt: currentTimestamp(), - }) - .where(eq(analyticsCohorts.id, cohort.id)); - if (memberPersonIds !== undefined) { - yield* tx - .delete(analyticsCohortMembers) - .where(eq(analyticsCohortMembers.cohortId, cohort.id)); - if (memberPersonIds.length > 0) { - yield* tx.insert(analyticsCohortMembers).values( - memberPersonIds.map((personId) => ({ - cohortId: cohort.id, - id: generateId("analyticsCohortMember"), - personId, - })), - ); - } - } - }), - ); - return yield* hydrateCohort(yield* loadCohortRow(cohort.id)); - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to update analytics cohort", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to update analytics cohort", - }), - ), - }), - ), - ); - - /** Soft-delete a cohort and remove its materialized membership. */ - const deleteCohort = Effect.fn("customAnalytics.deleteCohort")( - function* (input: { readonly id: string }) { - const cohort = yield* loadCohortRow(input.id); - yield* db.transaction((tx) => - Effect.gen(function* () { - yield* tx - .update(analyticsCohorts) - .set({ deletedAt: currentTimestamp(), updatedAt: currentTimestamp() }) - .where(eq(analyticsCohorts.id, cohort.id)); - yield* tx - .delete(analyticsCohortMembers) - .where(eq(analyticsCohortMembers.cohortId, cohort.id)); - }), - ); - return { deleted: true }; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to delete analytics cohort", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to delete analytics cohort", - }), - ), - }), - ), - ); - - /** List dashboards and their ordered insight placements. */ - const listDashboards = Effect.fn("customAnalytics.listDashboards")( - function* (input: { readonly projectId: string }) { - yield* getProject(input.projectId); - const rows = yield* db - .select() - .from(analyticsDashboards) - .where( - and( - eq(analyticsDashboards.projectId, input.projectId), - isNull(analyticsDashboards.deletedAt), - ), - ) - .orderBy(desc(analyticsDashboards.updatedAt)); - return { dashboards: yield* Effect.forEach(rows, hydrateDashboard) }; - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to list analytics dashboards", - }), - ), - ), - ), - ); - - /** Create an empty dashboard for a project. */ - const createDashboard = Effect.fn("customAnalytics.createDashboard")( - function* (input: { - readonly description?: string; - readonly name: string; - readonly projectId: string; - }) { - const project = yield* getProject(input.projectId); - const session = yield* AuthSession; - const id = generateId("analyticsDashboard"); - yield* db.insert(analyticsDashboards).values({ - createdBy: session?.user?.id ?? "system", - description: input.description, - id, - name: input.name, - organizationId: project.organizationId, - projectId: project.id, - }); - return yield* hydrateDashboard(yield* loadDashboardRow(id)); - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to create analytics dashboard", - }), - ), - ), - ), - ); - - /** Duplicate a dashboard and preserve its ordered card layout. */ - const duplicateDashboard = Effect.fn("customAnalytics.duplicateDashboard")( - function* (input: { readonly id: string; readonly name?: string }) { - const source = yield* loadDashboardRow(input.id); - const sourceItems = yield* db - .select() - .from(analyticsDashboardItems) - .where(eq(analyticsDashboardItems.dashboardId, source.id)) - .orderBy(asc(analyticsDashboardItems.position)); - const session = yield* AuthSession; - const id = generateId("analyticsDashboard"); - yield* db.transaction((tx) => - Effect.gen(function* () { - yield* tx.insert(analyticsDashboards).values({ - createdBy: session?.user?.id ?? "system", - description: source.description, - id, - name: input.name?.trim() || `${source.name} copy`, - organizationId: source.organizationId, - projectId: source.projectId, - }); - if (sourceItems.length > 0) { - yield* tx.insert(analyticsDashboardItems).values( - sourceItems.map((item) => ({ - dashboardId: id, - id: generateId("analyticsDashboardItem"), - layout: item.layout, - position: item.position, - sourceId: item.sourceId, - sourceType: item.sourceType, - })), - ); - } - }), - ); - return yield* hydrateDashboard(yield* loadDashboardRow(id)); - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to duplicate analytics dashboard", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to duplicate analytics dashboard", - }), - ), - }), - ), - ); - - /** Update dashboard metadata. */ - const updateDashboard = Effect.fn("customAnalytics.updateDashboard")( - function* (input: { - readonly description?: string | null; - readonly id: string; - readonly name?: string; - }) { - yield* loadDashboardRow(input.id); - const changes: { description?: string | null; name?: string } = {}; - if (input.description !== undefined) changes.description = input.description; - if (input.name !== undefined) changes.name = input.name; - yield* db - .update(analyticsDashboards) - .set({ - ...changes, - updatedAt: currentTimestamp(), - }) - .where(eq(analyticsDashboards.id, input.id)); - return yield* hydrateDashboard(yield* loadDashboardRow(input.id)); - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to update analytics dashboard", - }), - ), - ), - ), - ); - - /** Soft-delete a dashboard and its placements. */ - const deleteDashboard = Effect.fn("customAnalytics.deleteDashboard")( - function* (input: { readonly id: string }) { - yield* loadDashboardRow(input.id); - yield* db.transaction((tx) => - Effect.gen(function* () { - yield* tx - .update(analyticsDashboards) - .set({ deletedAt: currentTimestamp(), updatedAt: currentTimestamp() }) - .where(eq(analyticsDashboards.id, input.id)); - yield* tx - .delete(analyticsDashboardItems) - .where(eq(analyticsDashboardItems.dashboardId, input.id)); - }), - ); - return { deleted: true }; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to delete analytics dashboard", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to delete analytics dashboard", - }), - ), - }), - ), - ); - - /** Add or move a saved insight or VoidQL query on a dashboard. */ - const putDashboardItem = Effect.fn("customAnalytics.putDashboardItem")( - function* (input: { - readonly dashboardId: string; - readonly layout: AnalyticsDashboardItemLayoutType; - readonly position: number; - readonly source: - | { readonly id: string; readonly kind: "insight" } - | { readonly id: string; readonly kind: "voidql" }; - }) { - const dashboard = yield* loadDashboardRow(input.dashboardId); - if (input.source.kind === "insight") { - const insight = yield* loadInsightRow(input.source.id); - if (dashboard.projectId !== insight.projectId) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: input.source.id, - message: "Dashboard and insight must belong to the same project", - }), - ); - } - } else { - const [query] = yield* db - .select({ organizationId: analyticsSavedQuery.organizationId }) - .from(analyticsSavedQuery) - .where(eq(analyticsSavedQuery.id, input.source.id)) - .limit(1); - if (!query || query.organizationId !== dashboard.organizationId) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: input.source.id, - message: "Dashboard and saved query must belong to the same organization", - }), - ); - } - } - const [existing] = yield* db - .select() - .from(analyticsDashboardItems) - .where( - and( - eq(analyticsDashboardItems.dashboardId, input.dashboardId), - eq(analyticsDashboardItems.sourceType, input.source.kind), - eq(analyticsDashboardItems.sourceId, input.source.id), - ), - ) - .limit(1); - if (existing) { - yield* db - .update(analyticsDashboardItems) - .set({ - layout: input.layout, - position: input.position, - updatedAt: currentTimestamp(), - }) - .where(eq(analyticsDashboardItems.id, existing.id)); - } else { - yield* db.insert(analyticsDashboardItems).values({ - dashboardId: input.dashboardId, - id: generateId("analyticsDashboardItem"), - layout: input.layout, - position: input.position, - sourceId: input.source.id, - sourceType: input.source.kind, - }); - } - return yield* hydrateDashboard(dashboard); - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to update analytics dashboard item", - }), - ), - ), - ), - ); - - /** Atomically replace the display order for every card on a dashboard. */ - const reorderDashboardItems = Effect.fn("customAnalytics.reorderDashboardItems")( - function* (input: { - readonly dashboardId: string; - readonly itemIds: ReadonlyArray; - }) { - const dashboard = yield* loadDashboardRow(input.dashboardId); - const itemRows = yield* db - .select({ id: analyticsDashboardItems.id }) - .from(analyticsDashboardItems) - .where(eq(analyticsDashboardItems.dashboardId, input.dashboardId)); - const itemIds = [...new Set(input.itemIds)]; - const existingIds = new Set(itemRows.map((item) => item.id)); - if ( - itemIds.length !== input.itemIds.length || - itemIds.length !== existingIds.size || - itemIds.some((id) => !existingIds.has(id)) - ) { - return yield* Effect.fail( - new AnalyticsServiceError({ - cause: input.dashboardId, - message: "Dashboard card order must include every card exactly once", - }), - ); - } - yield* db.transaction((tx) => - Effect.forEach( - itemIds, - (id, position) => - tx - .update(analyticsDashboardItems) - .set({ position, updatedAt: currentTimestamp() }) - .where( - and( - eq(analyticsDashboardItems.dashboardId, input.dashboardId), - eq(analyticsDashboardItems.id, id), - ), - ), - { discard: true }, - ), - ); - return yield* hydrateDashboard(dashboard); - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to reorder analytics dashboard cards", - }), - ), - SqlError: (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: error.message, - message: "Failed to reorder analytics dashboard cards", - }), - ), - }), - ), - ); - - /** Remove a saved analytics card from a dashboard. */ - const removeDashboardItem = Effect.fn("customAnalytics.removeDashboardItem")( - function* (input: { readonly dashboardId: string; readonly itemId: string }) { - const dashboard = yield* loadDashboardRow(input.dashboardId); - yield* db - .delete(analyticsDashboardItems) - .where( - and( - eq(analyticsDashboardItems.dashboardId, input.dashboardId), - eq(analyticsDashboardItems.id, input.itemId), - ), - ); - return yield* hydrateDashboard(dashboard); - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", (error) => - Effect.fail( - new AnalyticsServiceError({ - cause: String(error.cause), - message: "Failed to remove analytics dashboard item", - }), - ), - ), - ), - ); - - return constant({ - createCohort, - createDashboard, - createInsight, - deleteCohort, - deleteDashboard, - deleteInsight, - duplicateDashboard, - listCohorts, - listDashboards, - listInsights, - putDashboardItem, - queryInsight, - queryPersons, - reorderDashboardItems, - removeDashboardItem, - updateDashboard, - updateCohort, - updateInsight, - }); - }), - }, -) { - static layer: Layer.Layer = Layer.effect( - CustomAnalyticsService, - )(CustomAnalyticsService.make); -} diff --git a/packages/core/src/services/analytics/clickhouse-accessor.ts b/packages/core/src/services/analytics/clickhouse-accessor.ts deleted file mode 100644 index e55aa7948..000000000 --- a/packages/core/src/services/analytics/clickhouse-accessor.ts +++ /dev/null @@ -1,1929 +0,0 @@ -/** - * ClickHouse data-access layer for the analytics service: the 12 metric query - * helpers consumed by `series-resolver`, exposed as {@link analyticsAccessor}. - * Each returns `ReadonlyArray` with the period column - * normalised to `Date`. - */ -import { DateTime, Effect, Option } from "effect"; -import { causeMessage, constant, numberOr } from "@voidhash/lib/lang"; - -import type { - AnalyticsDataPoint, - CompiledAnalyticsFilter, - TimeGranularity, - TimeRangeParams, -} from "../../domain/analytics/Analytics.ts"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import type { - AnalyticsActorType, - AnalyticsBreakdownType, - AnalyticsEventSeriesType, - AnalyticsFilterType, - FunnelsInsightQueryType, - LifecycleInsightQueryType, - PathsInsightQueryType, - RetentionInsightQueryType, - StickinessInsightQueryType, -} from "@voidhash/rpc"; -import type { SqlError } from "effect/unstable/sql/SqlError"; - -// Unqualified table names — the runtime Clickhouse client connects with the -// per-stage database (provisioned by `Clickhouse.Database`) as its default, -// so these resolve correctly without a hardcoded database prefix. -const CLICKHOUSE_EVENTS_FULL_TABLE = constant("events_v2"); -const CLICKHOUSE_PERSONS_FULL_TABLE = constant("persons_v1"); -const CLICKHOUSE_PENDING_OVERRIDES_FULL_TABLE = constant("person_identity_pending_overrides_v2"); - -const EVENT_ALIAS = "events"; -const OVERRIDES_ALIAS = "pending_overrides"; -const EVENT_TS = `${EVENT_ALIAS}.event_ts`; -const EVENT_PROPERTIES = `${EVENT_ALIAS}.event_properties`; - -// The pending-overrides columns are non-nullable `String` and ClickHouse runs -// with `join_use_nulls = 0`, so an unmatched LEFT JOIN row yields '' (empty -// string), not NULL. A plain `coalesce(overrides.col, events.col)` would stop at -// that '' and collapse every unmatched person into one empty-string key, so each -// override column is `nullIf(col, '')`'d first — turning the no-match '' back -// into NULL — before coalescing to the event's own id. -const effectivePersonIdExpression = `coalesce(nullIf(${OVERRIDES_ALIAS}.person_id, ''), ${EVENT_ALIAS}.person_id)`; -const effectiveDistinctIdExpression = `coalesce(nullIf(${OVERRIDES_ALIAS}.target_distinct_id, ''), ${EVENT_ALIAS}.distinct_id)`; - -const PENDING_OVERRIDES_SUBQUERY = ` -( - SELECT - project_id, - source_distinct_id, - target_distinct_id, - person_id - FROM ( - SELECT - project_id, - source_distinct_id, - target_distinct_id, - person_id, - is_deleted, - version, - changed_at - FROM ${CLICKHOUSE_PENDING_OVERRIDES_FULL_TABLE} - WHERE version > 0 - ORDER BY - project_id ASC, - source_distinct_id ASC, - version DESC, - changed_at DESC - LIMIT 1 BY project_id, source_distinct_id - ) - WHERE is_deleted = 0 -) AS ${OVERRIDES_ALIAS}`; - -// Columns the deduped `events` subquery (see `resolvedEventsFrom`) projects for -// the outer query, the pending-overrides JOIN, and the metric expressions. -// `processed_ts` is the latest-wins ORDER BY key — read off the base scan inside -// the subquery, so it does not need to be projected here. -const RESOLVED_EVENTS_COLUMNS = [ - "event_id", - "event_name", - "event_ts", - "project_id", - "distinct_id", - "person_id", - "event_properties", -].join(", "); - -const RESOLVED_EVENTS_JOIN = `LEFT JOIN ${PENDING_OVERRIDES_SUBQUERY} -ON ${OVERRIDES_ALIAS}.project_id = ${EVENT_ALIAS}.project_id -AND ${OVERRIDES_ALIAS}.source_distinct_id = ${EVENT_ALIAS}.distinct_id`; - -const getDateTruncExpression = (column: string, granularity: TimeGranularity): string => { - switch (granularity) { - case "hour": - return `toStartOfHour(${column})`; - case "day": - return `toDate(${column})`; - case "week": - return `toStartOfWeek(${column}, 1)`; - case "month": - return `toStartOfMonth(${column})`; - case "quarter": - return `toStartOfQuarter(${column})`; - case "year": - return `toStartOfYear(${column})`; - } -}; - -interface AnalyticsRow { - period: string | Date; - total: string | number | null; -} - -/** Normalises a ClickHouse `DateTime` string to an ISO-8601 UTC instant. */ -const withDateTimeSeparator = (trimmed: string): string => { - if (trimmed.includes("T")) return trimmed; - return trimmed.replace(" ", "T"); -}; - -const toUtcInstantString = (trimmed: string): string => { - const normalized = withDateTimeSeparator(trimmed); - if (/(?:Z|[+-]\d{2}:\d{2})$/.test(normalized)) return normalized; - return `${normalized}Z`; -}; - -const parsePeriod = (value: unknown): Date | null => { - if (value instanceof Date) { - if (Number.isNaN(value.getTime())) return null; - return value; - } - if (typeof value === "string") { - const trimmed = value.trim(); - if (trimmed.length === 0) return null; - return Option.match(DateTime.make(toUtcInstantString(trimmed)), { - onNone: () => null, - onSome: (instant) => DateTime.toDateUtc(instant), - }); - } - return null; -}; - -const parseRowsToDataPoints = (rows: ReadonlyArray): AnalyticsDataPoint[] => - rows.flatMap((row) => { - const timestamp = parsePeriod(row.period); - if (!timestamp) return []; - return [{ timestamp, value: numberOr(Number(row.total ?? 0), 0) }]; - }); - -const jsonString = (...keys: readonly string[]): string => - `coalesce(${keys - .map((key) => `nullIf(JSONExtractString(${EVENT_PROPERTIES}, '${key}'), '')`) - .join(", ")}, '')`; - -const jsonNumber = (...keys: readonly string[]): string => - `coalesce(${keys - .map((key) => `nullIf(JSONExtractFloat(${EVENT_PROPERTIES}, '${key}'), 0)`) - .join(", ")}, 0)`; - -const jsonBool = (...keys: readonly string[]): string => - `greatest(${keys.map((key) => `JSONExtractBool(${EVENT_PROPERTIES}, '${key}')`).join(", ")})`; - -// USD-only. A row with no FX rate at write time has `amount_usd` NULL; we must -// NOT fall back to the raw original-currency `amount` (e.g. summing £10 as $10), -// so an FX-less row contributes 0 to USD revenue — a bounded under-count rather -// than a silent wrong-currency miscount. Carry-forward FX at write time -// (`FxRateService.getUsdRate`) keeps almost every new row valued, so the -// under-count is negligible. -const amountCentsExpression = jsonNumber("amount_usd", "amountUsd"); -const productIdExpression = jsonString("product_id", "productId", "product.id"); -const providerEnvironmentExpression = jsonNumber("provider_environment", "providerEnvironment"); -const subscriptionStatusExpression = jsonNumber("subscription_status", "subscriptionStatus"); -const subscriptionIdExpression = `coalesce( - nullIf(JSONExtractString(${EVENT_PROPERTIES}, 'subscription_id'), ''), - nullIf(JSONExtractString(${EVENT_PROPERTIES}, 'subscriptionId'), ''), - nullIf(JSONExtractString(${EVENT_PROPERTIES}, 'provider_subscription_id'), ''), - nullIf(JSONExtractString(${EVENT_PROPERTIES}, 'providerSubscriptionId'), ''), - nullIf(JSONExtractString(${EVENT_PROPERTIES}, 'store_subscription_id'), ''), - nullIf(JSONExtractString(${EVENT_PROPERTIES}, 'storeSubscriptionId'), ''), - ${EVENT_ALIAS}.event_id -)`; - -/** - * ClickHouse's `DateTime` named parameter expects `YYYY-MM-DD HH:MM:SS`. The - * Web client serialises a JS `Date` as ISO with milliseconds otherwise, which - * the binding parser rejects. - */ -const toClickhouseDateTime = (date: Date): string => - date - .toISOString() - .replace("T", " ") - .replace(/\.\d+Z$/, ""); - -/** - * Build the optional event-property filter clause as a composed SQL fragment. - * Each active filter contributes an `AND IN ` term - * (the expression is raw SQL, the values bound as a typed ClickHouse array - * parameter); inactive filters contribute nothing. Returns an empty fragment - * when no filters apply, so it can be interpolated unconditionally into the - * surrounding query. - */ -const buildEventFilters = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - filters: CompiledAnalyticsFilter, -) => { - const terms = [ - { values: filters.productIds, expression: productIdExpression, kind: constant("String") }, - { - values: filters.providerEnvironments, - expression: providerEnvironmentExpression, - kind: constant("Float64"), - }, - { - values: filters.subscriptionStatuses, - expression: subscriptionStatusExpression, - kind: constant("Float64"), - }, - ]; - return terms.reduce((acc, term) => { - if (!term.values || term.values.length === 0) return acc; - return ch`${acc} - AND ${ch.literal(term.expression)} IN ${ch.param(`Array(${term.kind})`, term.values)}`; - }, ch``); -}; - -/** The composed SQL fragment type produced by the `ch` tagged template. */ -type SqlFragment = ReturnType; - -/** - * Lazily selects one of two branches. Stands in for the conditional expressions - * this module used to build its SQL fragments with (banned by the lint preset) - * while keeping the unchosen branch unevaluated. - */ -const branch = (condition: boolean, onTrue: () => A, onFalse: () => A): A => { - if (condition) return onTrue(); - return onFalse(); -}; - -/** - * Applies `onDefined` to a present optional value, otherwise yields - * `onAbsent()` — the fragment-building counterpart of `Option.match`. - */ -const whenDefined = ( - value: T | undefined, - onDefined: (value: T) => A, - onAbsent: () => A, -): A => { - if (value === undefined) return onAbsent(); - return onDefined(value); -}; - -/** - * `FROM () AS events` — collapses the raw `events_v2` - * MergeTree (which can hold more than one row for a single `event_id` after a - * retry, replay, or concurrent flush) down to the newest row per `event_id`, - * ordered by `processed_ts` (the ingestion timestamp). Defense-in-depth so the - * money sums never double-count a redelivered event; count/`countDistinct` - * metrics benefit too. - * - * The caller's partition-pruning predicate (`project_id` / `event_ts` range / - * `event_name`) is pushed INTO the inner scan via `innerWhere` so `LIMIT 1 BY` - * only walks the matching partitions, not the whole table. The RLS - * `SQL_organization_id` setting is a whole-statement ClickHouse setting, so it - * still filters every `events_v2` access here — the subquery does not bypass the - * row policy. Inside the subquery the columns are unqualified (no `events.` - * alias yet); the outer query then references them through the `events` alias. - */ -const resolvedEventsFrom = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - innerWhere: ReturnType, -) => - ch`FROM ( - SELECT ${ch.literal(RESOLVED_EVENTS_COLUMNS)} - FROM ${ch.literal(CLICKHOUSE_EVENTS_FULL_TABLE)} - WHERE ${innerWhere} - ORDER BY processed_ts DESC - LIMIT 1 BY event_id - ) AS ${ch.literal(EVENT_ALIAS)}`; - -const moneyPoints = (points: ReadonlyArray): AnalyticsDataPoint[] => - points.map((point) => ({ ...point, value: point.value / 100 })); - -interface EventMetricInput { - readonly aggregateExpression: string; - readonly eventNames: readonly string[]; - readonly extraWhere?: string; - readonly filters: CompiledAnalyticsFilter; - readonly organizationId: string; - readonly params: TimeRangeParams; -} - -/** - * Per-query setting the readonly ClickHouse user's row policies read. Passing - * it scopes every analytics read to the caller's tenant; omitting it would - * match no rows (fail-closed). - */ -const tenantSettings = (organizationId: string): Record => ({ - SQL_organization_id: organizationId, -}); - -const eventMetric = (input: EventMetricInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const period = getDateTruncExpression(EVENT_TS, input.params.granularity); - const eventFilters = buildEventFilters(ch, input.filters); - // The project/time/event-name predicate is the partition-pruning filter, so - // it lives inside the dedup subquery (before `LIMIT 1 BY`). The optional - // event-property filters (`eventFilters`) and `extraWhere` stay in the outer - // WHERE: they read `event_properties`, and applying them to the already- - // deduped row is correct (a true duplicate carries identical properties) and - // cheaper than running JSONExtract on the pre-dedup rows. - const dedupedEvents = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - AND event_ts >= ${ch.param("DateTime", toClickhouseDateTime(input.params.startDate))} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - AND event_name IN ${ch.param("Array(String)", input.eventNames)}`, - ); - - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - ${ch.literal(period)} AS period, - ${ch.literal(input.aggregateExpression)} AS total - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE 1 = 1 - ${eventFilters} - ${ch.literal(input.extraWhere ?? "")} - GROUP BY period - ORDER BY period ASC - `, - tenantSettings(input.organizationId), - ); - - return parseRowsToDataPoints(rows); - }); - -export interface EventTrendQueryInput extends AnalyticsActorQueryInput { - readonly aggregation: AnalyticsEventSeriesType["aggregation"]; - readonly aggregateOverRange?: boolean; - readonly breakdown?: AnalyticsBreakdownType; - readonly eventNames: readonly string[]; - readonly filters: CompiledAnalyticsFilter; - readonly mathProperty?: string; - readonly propertyFilter?: AnalyticsFilterType; - readonly organizationId: string; - readonly params: TimeRangeParams; -} - -export interface EventTrendSeriesGroup { - readonly breakdownValue?: string; - readonly points: AnalyticsDataPoint[]; -} - -interface AnalyticsActorQueryInput { - readonly actor?: AnalyticsActorType; - readonly cohortPersonIds?: readonly string[]; -} - -interface EventTrendRow extends AnalyticsRow { - breakdown: string; -} - -const customEventField = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - field: string, -): ReturnType => { - if (field === "event.name") return ch`${ch.literal(`${EVENT_ALIAS}.event_name`)}`; - if (field === "person.id") { - return ch`${ch.literal( - `coalesce(${effectivePersonIdExpression}, ${effectiveDistinctIdExpression})`, - )}`; - } - return ch`JSONExtractString( - ${ch.literal(EVENT_PROPERTIES)}, - ${ch.param("String", field.slice("event.properties.".length))} - )`; -}; - -const analyticsActorKey = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - actor: AnalyticsActorType | undefined, -): SqlFragment => { - if (actor?.kind === "group") { - return ch`nullIf(JSONExtractString( - ${ch.literal(EVENT_PROPERTIES)}, - ${ch.param("String", actor.property)} - ), '')`; - } - return ch`${ch.literal( - `coalesce(${effectivePersonIdExpression}, ${effectiveDistinctIdExpression})`, - )}`; -}; - -const analyticsCohortFilter = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - cohortPersonIds: readonly string[] | undefined, -): SqlFragment => { - if (cohortPersonIds === undefined) return ch`1 = 1`; - if (cohortPersonIds.length === 0) return ch`0 = 1`; - return ch`${ch.literal(effectivePersonIdExpression)} IN ${ch.param( - "Array(String)", - cohortPersonIds, - )}`; -}; - -const analyticsActorFilter = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - input: AnalyticsActorQueryInput, -): SqlFragment => { - const actorKey = analyticsActorKey(ch, input.actor); - const cohortFilter = analyticsCohortFilter(ch, input.cohortPersonIds); - return ch`notEmpty(toString(${actorKey})) AND ${cohortFilter}`; -}; - -const stringFilterValue = (value: unknown): string => causeMessage(value ?? ""); - -const OPERATOR_BY_FILTER_TYPE = constant({ and: "AND", or: "OR" }); - -/** - * Narrows a filter node to its leaf predicate. The `and` / `or` member of - * {@link AnalyticsFilterType} carries a two-literal `type`, which control-flow - * analysis cannot discriminate away on its own. - */ -const isFilterPredicate = ( - filter: AnalyticsFilterType, -): filter is Extract => - filter.type === "predicate"; - -/** Coerces a predicate's value to the string array an `IN` / `NOT IN` term binds. */ -const stringFilterValues = (value: unknown): string[] => { - if (Array.isArray(value)) return value.map(stringFilterValue); - return []; -}; - -const buildCustomEventFilter = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - filter: AnalyticsFilterType, -): ReturnType => { - if (filter.type === "and" || filter.type === "or") { - const operator = OPERATOR_BY_FILTER_TYPE[filter.type]; - return filter.filters - .reduce((acc, child, index) => { - if (index === 0) return ch`(${buildCustomEventFilter(ch, child)}`; - return ch`${acc} ${ch.literal(operator)} ${buildCustomEventFilter(ch, child)}`; - }, ch``) - .pipe((fragment) => ch`${fragment})`); - } - if (filter.type === "not") { - return ch`NOT (${buildCustomEventFilter(ch, filter.filter)})`; - } - - if (!isFilterPredicate(filter)) return ch`0 = 1`; - - const predicate = filter; - const expression = customEventField(ch, predicate.field); - switch (predicate.op) { - case "eq": - return ch`${expression} = ${ch.param("String", stringFilterValue(predicate.value))}`; - case "neq": - return ch`${expression} != ${ch.param("String", stringFilterValue(predicate.value))}`; - case "in": - return ch`${expression} IN ${ch.param("Array(String)", stringFilterValues(predicate.value))}`; - case "not_in": - return ch`${expression} NOT IN ${ch.param( - "Array(String)", - stringFilterValues(predicate.value), - )}`; - case "contains": - return ch`positionCaseInsensitive( - ${expression}, - ${ch.param("String", stringFilterValue(predicate.value))} - ) > 0`; - case "exists": - return ch`notEmpty(${expression})`; - case "gt": - case "gte": - case "lt": - case "lte": - return ch`0 = 1`; - } - return ch`0 = 1`; -}; - -/** Renders a breakdown's sort order as the SQL keyword. */ -const sortDirection = (order: AnalyticsBreakdownType["order"]): string => { - if (order === "asc") return "ASC"; - return "DESC"; -}; - -/** Only breakdown queries carry a `breakdownValue` on their series groups. */ -const breakdownValueEntry = ( - breakdown: AnalyticsBreakdownType | undefined, - breakdownValue: string, -) => { - if (breakdown === undefined) return {}; - return { breakdownValue }; -}; - -/** Maps a trend series' aggregation to its ClickHouse aggregate expression. */ -const eventTrendAggregate = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - aggregation: EventTrendQueryInput["aggregation"], - actorKey: SqlFragment, - propertyExpression: SqlFragment, -): SqlFragment => { - switch (aggregation) { - case "unique_users": - return ch`countDistinct(${actorKey})`; - case "property_sum": - return ch`sum(${propertyExpression})`; - case "property_average": - return ch`avg(${propertyExpression})`; - case "property_minimum": - return ch`min(${propertyExpression})`; - case "property_maximum": - return ch`max(${propertyExpression})`; - case "property_median": - return ch`quantileExact(0.5)(${propertyExpression})`; - case "property_p75": - return ch`quantileExact(0.75)(${propertyExpression})`; - case "property_p90": - return ch`quantileExact(0.9)(${propertyExpression})`; - case "property_p95": - return ch`quantileExact(0.95)(${propertyExpression})`; - case "property_p99": - return ch`quantileExact(0.99)(${propertyExpression})`; - default: - return ch`count()`; - } -}; - -/** - * Query a custom event series using the identity-stitched analytics event view. - * Number displays can aggregate the whole range into one point without summing per-bucket users. - */ -export const getEventTrendSeries = (input: EventTrendQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const actorKey = analyticsActorKey(ch, input.actor); - const actorFilter = analyticsActorFilter(ch, input); - const period = branch( - input.aggregateOverRange === true, - () => "toDateTime('1970-01-01 00:00:00')", - () => getDateTruncExpression(EVENT_TS, input.params.granularity), - ); - const propertyExpression = ch`toFloat64OrNull(JSONExtractRaw( - ${ch.literal(EVENT_PROPERTIES)}, - ${ch.param("String", input.mathProperty ?? "")} - ))`; - const aggregateExpression = eventTrendAggregate( - ch, - input.aggregation, - actorKey, - propertyExpression, - ); - const breakdownExpression = whenDefined( - input.breakdown, - (breakdown) => customEventField(ch, breakdown.field), - () => ch`${ch.param("String", "")}`, - ); - const customFilter = whenDefined( - input.propertyFilter, - (filter) => buildCustomEventFilter(ch, filter), - () => ch`1 = 1`, - ); - const perPeriodLimit = whenDefined( - input.breakdown, - (breakdown) => Math.min((breakdown.limit ?? 10) * 4, 400), - () => 1, - ); - const dedupedEvents = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - AND event_ts >= ${ch.param("DateTime", toClickhouseDateTime(input.params.startDate))} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - AND event_name IN ${ch.param("Array(String)", input.eventNames)}`, - ); - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - ${ch.literal(period)} AS period, - ${breakdownExpression} AS breakdown, - ${aggregateExpression} AS total - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${customFilter} - AND ${actorFilter} - GROUP BY period, breakdown - ORDER BY period ASC, total DESC - LIMIT ${ch.param("UInt32", perPeriodLimit)} BY period - `, - tenantSettings(input.organizationId), - ); - - const grouped = new Map(); - for (const row of rows) { - const points = parseRowsToDataPoints([row]); - if (points.length === 0) continue; - const key = whenDefined( - input.breakdown, - () => row.breakdown, - () => "", - ); - const existing = grouped.get(key); - if (existing) existing.push(points[0]); - else grouped.set(key, points); - } - - const breakdown = input.breakdown; - const groups = [...grouped.entries()].map(([breakdownValue, points]) => ({ - ...breakdownValueEntry(breakdown, breakdownValue), - points, - })); - if (!breakdown) return groups; - - const direction = branch( - breakdown.order === "asc", - () => 1, - () => -1, - ); - return groups - .sort( - (left, right) => - direction * - (left.points.reduce((sum, point) => sum + point.value, 0) - - right.points.reduce((sum, point) => sum + point.value, 0)), - ) - .slice(0, breakdown.limit ?? 10); - }); - -export interface EventPersonDrilldownQueryInput { - readonly cohortPersonIds?: readonly string[]; - readonly eventNames: readonly string[]; - readonly filters?: AnalyticsFilterType; - readonly group?: { readonly property: string; readonly value: string }; - readonly limit: number; - readonly organizationId: string; - readonly params: Pick; - readonly projectId: string; -} - -export interface EventPersonDrilldownRow { - readonly eventCount: number; - readonly lastSeenAt: Date; - readonly personId: string; -} - -interface EventPersonDrilldownSqlRow { - event_count: number | string; - last_seen_at: Date | string; - person_id: string; -} - -/** Query people who performed a selected event series in a bounded insight window. */ -export const getEventPersonDrilldown = (input: EventPersonDrilldownQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const propertyFilter = whenDefined( - input.filters, - (filters) => buildCustomEventFilter(ch, filters), - () => ch`1 = 1`, - ); - const cohortFilter = analyticsActorFilter(ch, { - actor: { kind: "person" }, - cohortPersonIds: input.cohortPersonIds, - }); - const groupFilter = whenDefined( - input.group, - (group) => - ch`${customEventField(ch, `event.properties.${group.property}`)} = ${ch.param( - "String", - group.value, - )}`, - () => ch`1 = 1`, - ); - const dedupedEvents = resolvedEventsFrom( - ch, - ch`project_id = ${ch.param("String", input.projectId)} - AND event_ts >= ${ch.param("DateTime", toClickhouseDateTime(input.params.startDate))} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - AND event_name IN ${ch.param("Array(String)", input.eventNames)}`, - ); - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - ${ch.literal(effectivePersonIdExpression)} AS person_id, - count() AS event_count, - max(${ch.literal(EVENT_TS)}) AS last_seen_at - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE notEmpty(person_id) - AND ${propertyFilter} - AND ${cohortFilter} - AND ${groupFilter} - GROUP BY person_id - ORDER BY last_seen_at DESC, event_count DESC, person_id ASC - LIMIT ${ch.param("UInt8", input.limit)} - `, - tenantSettings(input.organizationId), - ); - - return rows.flatMap((row) => { - const lastSeenAt = parsePeriod(row.last_seen_at); - if (!lastSeenAt) return []; - return [{ eventCount: Number(row.event_count ?? 0), lastSeenAt, personId: row.person_id }]; - }); - }); - -export interface EventFunnelQueryInput extends AnalyticsActorQueryInput { - readonly breakdown?: AnalyticsBreakdownType; - readonly breakdownAttributionStep?: number; - readonly conversionWindowSeconds: number; - readonly filters: CompiledAnalyticsFilter; - readonly order: FunnelsInsightQueryType["order"]; - readonly organizationId: string; - readonly params: Pick; - readonly steps: FunnelsInsightQueryType["steps"]; -} - -const joinSqlFragments = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - fragments: ReadonlyArray>, - separator = ", ", -): ReturnType => - fragments.reduce((acc, fragment, index) => { - if (index === 0) return fragment; - return ch`${acc}${ch.literal(separator)}${fragment}`; - }, ch``); - -interface EventFunnelBreakdownCounts { - readonly breakdownValue: string; - readonly counts: number[]; -} - -const queryEventFunnelRows = ( - input: EventFunnelQueryInput, - withBreakdown: boolean, -): Effect.Effect< - ReadonlyArray>, - SqlError, - ClickhouseWebClient.ClickhouseWebClient -> => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const breakdown = branch( - withBreakdown, - () => input.breakdown, - () => undefined, - ); - const conversionWindowMilliseconds = input.conversionWindowSeconds * 1_000; - const allEventNames = [...new Set(input.steps.flatMap((step) => [...step.eventNames]))]; - const eventNameScope = branch( - input.order === "strict", - () => ch``, - () => ch`AND event_name IN ${ch.param("Array(String)", allEventNames)}`, - ); - const dedupedEvents = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - AND event_ts >= ${ch.param("DateTime", toClickhouseDateTime(input.params.startDate))} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - ${eventNameScope}`, - ); - const actorKey = analyticsActorKey(ch, input.actor); - const actorFilter = analyticsActorFilter(ch, input); - const stepConditions = input.steps.map((step) => { - const propertyFilter = whenDefined( - step.filters, - (filters) => buildCustomEventFilter(ch, filters), - () => ch`1 = 1`, - ); - return ch`( - ${ch.literal(`${EVENT_ALIAS}.event_name`)} IN ${ch.param("Array(String)", step.eventNames)} - AND ${propertyFilter} - )`; - }); - const attributionIndex = Math.min( - Math.max((input.breakdownAttributionStep ?? 1) - 1, 0), - input.steps.length - 1, - ); - const attributionCondition = stepConditions[attributionIndex] ?? ch`0 = 1`; - const breakdownAggregate = whenDefined( - breakdown, - (value) => - ch`argMinIf( - toString(${customEventField(ch, value.field)}), - ${ch.literal(EVENT_TS)}, - ${attributionCondition} - )`, - () => undefined, - ); - const breakdownProjection = whenDefined( - breakdownAggregate, - (aggregate) => ch`, ${aggregate} AS breakdown_value`, - () => ch``, - ); - const breakdownSelect = whenDefined( - breakdown, - () => ch`breakdown_value,`, - () => ch``, - ); - const breakdownGrouping = whenDefined( - breakdown, - (value) => - ch`GROUP BY breakdown_value - ORDER BY ${ch.literal(`step_${input.steps.length - 1}`)} - ${ch.literal(sortDirection(value.order))}, - breakdown_value ASC - LIMIT ${ch.param("UInt32", value.limit ?? 10)}`, - () => ch``, - ); - - if (input.order === "any") { - const timeArrays = stepConditions.map( - (condition, index) => - ch`groupArrayIf( - toUInt64(toUnixTimestamp64Milli(${ch.literal(EVENT_TS)})), - ${condition} - ) AS ${ch.literal(`times_${index}`)}`, - ); - const counts = input.steps.map((_, stepIndex) => { - const aliases = Array.from({ length: stepIndex + 1 }, (_, index) => `times_${index}`); - if (stepIndex === 0) { - return ch`countIf(notEmpty(${ch.literal(aliases[0] ?? "times_0")})) AS ${ch.literal( - "step_0", - )}`; - } - const candidates = joinSqlFragments( - ch, - aliases.map((alias) => ch`${ch.literal(alias)}`), - ); - const windowChecks = joinSqlFragments( - ch, - aliases.map( - (alias) => - ch`arrayExists( - candidate -> candidate >= window_start - AND candidate <= window_start + ${ch.param("UInt64", conversionWindowMilliseconds)}, - ${ch.literal(alias)} - )`, - ), - " AND ", - ); - return ch`countIf( - arrayExists( - window_start -> (${windowChecks}), - arrayConcat(${candidates}) - ) - ) AS ${ch.literal(`step_${stepIndex}`)}`; - }); - return yield* ch.withClickhouseSettings( - ch>` - SELECT ${breakdownSelect} ${joinSqlFragments(ch, counts)} - FROM ( - SELECT - ${actorKey} AS person_key, - ${joinSqlFragments(ch, timeArrays)} - ${breakdownProjection} - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${actorFilter} - GROUP BY person_key - ) - ${breakdownGrouping} - `, - tenantSettings(input.organizationId), - ); - } - - const modes = branch( - input.order === "strict", - () => - ch`${ch.param("UInt64", conversionWindowMilliseconds)}, 'strict_deduplication', 'strict_order'`, - () => ch`${ch.param("UInt64", conversionWindowMilliseconds)}, 'strict_deduplication'`, - ); - const funnelLevel = ch`windowFunnel(${modes})( - toUInt64(toUnixTimestamp64Milli(${ch.literal(EVENT_TS)})), - ${joinSqlFragments(ch, stepConditions)} - )`; - const counts = input.steps.map( - (_, index) => - ch`countIf(level >= ${ch.param("UInt8", index + 1)}) AS ${ch.literal(`step_${index}`)}`, - ); - return yield* ch.withClickhouseSettings( - ch>` - SELECT ${breakdownSelect} ${joinSqlFragments(ch, counts)} - FROM ( - SELECT - ${actorKey} AS person_key, - ${funnelLevel} AS level - ${breakdownProjection} - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${actorFilter} - GROUP BY person_key - ) - ${breakdownGrouping} - `, - tenantSettings(input.organizationId), - ); - }); - -const funnelCountsFromRow = ( - row: Record | undefined, - stepCount: number, -): number[] => Array.from({ length: stepCount }, (_, index) => Number(row?.[`step_${index}`] ?? 0)); - -/** Query overall step reach counts for an identity-stitched event funnel. */ -export const getEventFunnelCounts = (input: EventFunnelQueryInput) => - queryEventFunnelRows(input, false).pipe( - Effect.map((rows) => funnelCountsFromRow(rows[0], input.steps.length)), - ); - -/** Query attributed step reach counts for the top funnel breakdown values. */ -export const getEventFunnelBreakdownCounts = ( - input: EventFunnelQueryInput, -): Effect.Effect< - ReadonlyArray, - SqlError, - ClickhouseWebClient.ClickhouseWebClient -> => { - if (!input.breakdown) return Effect.succeed([]); - return queryEventFunnelRows(input, true).pipe( - Effect.map((rows) => - rows.map((row) => ({ - breakdownValue: String(row.breakdown_value ?? ""), - counts: funnelCountsFromRow(row, input.steps.length), - })), - ), - ); -}; - -export interface EventRetentionQueryInput extends AnalyticsActorQueryInput { - readonly cumulative: boolean; - readonly filters: CompiledAnalyticsFilter; - readonly intervals: number; - readonly organizationId: string; - readonly params: Pick; - readonly period: RetentionInsightQueryType["period"]; - readonly retentionType: "first_time" | "recurring"; - readonly returning: RetentionInsightQueryType["returning"]; - readonly start: RetentionInsightQueryType["start"]; -} - -export interface EventRetentionCohort { - readonly cohortSize: number; - readonly cohortStart: Date; - readonly counts: number[]; -} - -interface EventRetentionRow extends Record { - cohort_size: string | number; - cohort_start: string | Date; -} - -/** Query recurring or first-time identity-stitched retention cohorts. */ -export const getEventRetentionCohorts = (input: EventRetentionQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const periodExpression = getDateTruncExpression(EVENT_TS, input.period); - const actorKey = analyticsActorKey(ch, input.actor); - const actorFilter = analyticsActorFilter(ch, input); - const startFilter = whenDefined( - input.start.filters, - (filters) => buildCustomEventFilter(ch, filters), - () => ch`1 = 1`, - ); - const returningFilter = whenDefined( - input.returning.filters, - (filters) => buildCustomEventFilter(ch, filters), - () => ch`1 = 1`, - ); - const startFrom = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - ${branch( - input.retentionType === "recurring", - () => - ch`AND event_ts >= ${ch.param( - "DateTime", - toClickhouseDateTime(input.params.startDate), - )}`, - () => ch``, - )} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - AND event_name IN ${ch.param("Array(String)", [...input.start.eventNames])}`, - ); - const returningFrom = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - AND event_ts >= ${ch.param("DateTime", toClickhouseDateTime(input.params.startDate))} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - AND event_name IN ${ch.param("Array(String)", [...input.returning.eventNames])}`, - ); - const starts = branch( - input.retentionType === "recurring", - () => ch` - SELECT - ${actorKey} AS person_key, - ${ch.literal(periodExpression)} AS cohort_start - ${startFrom} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${startFilter} AND ${actorFilter} - GROUP BY person_key, cohort_start - `, - () => ch` - SELECT - ${actorKey} AS person_key, - min(${ch.literal(periodExpression)}) AS cohort_start - ${startFrom} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${startFilter} AND ${actorFilter} - GROUP BY person_key - HAVING cohort_start >= ${ch.param( - "DateTime", - toClickhouseDateTime(input.params.startDate), - )} - `, - ); - const returns = ch` - SELECT - ${actorKey} AS person_key, - ${ch.literal(periodExpression)} AS return_period - ${returningFrom} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${returningFilter} AND ${actorFilter} - GROUP BY person_key, return_period - `; - const intervalCounts = Array.from({ length: input.intervals }, (_, interval) => { - const intervalCondition = branch( - input.cumulative && interval > 0, - () => - ch`dateDiff(${ch.param("String", input.period)}, starts.cohort_start, returns.return_period) - >= ${ch.param("UInt8", interval)}`, - () => - ch`dateDiff(${ch.param("String", input.period)}, starts.cohort_start, returns.return_period) - = ${ch.param("UInt8", interval)}`, - ); - return ch`uniqExactIf( - starts.person_key, - notEmpty(returns.person_key) AND ${intervalCondition} - ) AS ${ch.literal(`interval_${interval}`)}`; - }); - const rows = yield* ch.withClickhouseSettings( - ch` - WITH - starts AS (${starts}), - returns AS (${returns}) - SELECT - starts.cohort_start AS cohort_start, - uniqExact(starts.person_key) AS cohort_size, - ${joinSqlFragments(ch, intervalCounts)} - FROM starts - LEFT JOIN returns - ON returns.person_key = starts.person_key - AND returns.return_period >= starts.cohort_start - AND dateDiff( - ${ch.param("String", input.period)}, - starts.cohort_start, - returns.return_period - ) < ${ch.param("UInt8", input.intervals)} - GROUP BY cohort_start - ORDER BY cohort_start ASC - `, - tenantSettings(input.organizationId), - ); - - return rows.flatMap((row) => { - const cohortStart = parsePeriod(row.cohort_start); - if (!cohortStart) return []; - return [ - { - cohortSize: Number(row.cohort_size ?? 0), - cohortStart, - counts: Array.from({ length: input.intervals }, (_, interval) => - Number(row[`interval_${interval}`] ?? 0), - ), - }, - ]; - }); - }); - -export interface EventPathsQueryInput extends AnalyticsActorQueryInput { - readonly definition: PathsInsightQueryType; - readonly filters: CompiledAnalyticsFilter; - readonly organizationId: string; - readonly params: Pick; -} - -export interface EventPathLink { - readonly averageTransitionSeconds: number; - readonly count: number; - readonly source: string; - readonly sourceStep: number; - readonly target: string; - readonly targetStep: number; -} - -interface EventPathRow { - average_transition_seconds: string | number; - count: string | number; - source: string; - source_step: string | number; - target: string; - target_step: string | number; -} - -/** Query adjacent event transitions from bounded, identity-stitched user sessions. */ -export const getEventPathLinks = (input: EventPathsQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const definition = input.definition; - const includeEvents = branch( - definition.eventNames.length > 0, - () => ch`AND event_name IN ${ch.param("Array(String)", [...definition.eventNames])}`, - () => ch``, - ); - const dedupedEvents = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - AND event_ts >= ${ch.param("DateTime", toClickhouseDateTime(input.params.startDate))} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - ${includeEvents}`, - ); - const actorKey = analyticsActorKey(ch, input.actor); - const actorFilter = analyticsActorFilter(ch, input); - const pathItem = branch( - definition.pathItem === "screen_name", - () => ch`${ch.literal(jsonString("$screen_name", "screen_name", "screenName"))}`, - () => ch`${ch.literal(`${EVENT_ALIAS}.event_name`)}`, - ); - const propertyFilter = whenDefined( - definition.filters, - (filters) => buildCustomEventFilter(ch, filters), - () => ch`1 = 1`, - ); - const excludeEventNames = definition.excludeEventNames; - const excludePathItems = branch( - (excludeEventNames?.length ?? 0) > 0, - () => ch`AND ${pathItem} NOT IN ${ch.param("Array(String)", [...(excludeEventNames ?? [])])}`, - () => ch``, - ); - const sessionGapSeconds = definition.sessionGapSeconds ?? 1_800; - const orderedEvents = ch` - SELECT - ${actorKey} AS person_key, - ${pathItem} AS path_item, - ${ch.literal(EVENT_TS)} AS path_time, - ${ch.literal(`${EVENT_ALIAS}.event_id`)} AS event_id - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${propertyFilter} AND ${actorFilter} ${excludePathItems} - ORDER BY person_key ASC, path_time ASC, event_id ASC - `; - const sessions = ch` - SELECT - person_key, - arraySplit( - item -> item.3 > ${ch.param("UInt32", sessionGapSeconds)}, - arrayZip( - path_items, - path_times, - arrayDifference(arrayMap(value -> toUInt64(toUnixTimestamp(value)), path_times)) - ) - ) AS sessions - FROM ( - SELECT - person_key, - groupArray(path_item) AS path_items, - groupArray(path_time) AS path_times - FROM (${orderedEvents}) - WHERE notEmpty(path_item) - GROUP BY person_key - ) - `; - const compactPath = branch( - definition.collapseRepeated === false, - () => ch`session`, - () => ch`arrayFilter( - (item, item_index) -> item_index = 1 - OR tupleElement(item, 1) != tupleElement(session[item_index - 1], 1), - session, - arrayEnumerate(session) - )`, - ); - const compactedPaths = ch` - SELECT - person_key, - ${compactPath} AS compact_path - FROM (${sessions}) - ARRAY JOIN sessions AS session - WHERE length(session) >= 2 - `; - const startEventName = definition.startEventName; - const endEventName = definition.endEventName; - const hasStart = Boolean(startEventName); - const hasEnd = Boolean(endEventName); - const startIndex = branch( - hasStart, - () => ch`indexOf(path_names, ${ch.param("String", startEventName ?? "")})`, - () => ch`1`, - ); - const endIndexWithinStart = branch( - hasStart, - () => ch`if( - indexOf( - arraySlice(path_names, start_index), - ${ch.param("String", endEventName ?? "")} - ) > 0, - indexOf( - arraySlice(path_names, start_index), - ${ch.param("String", endEventName ?? "")} - ) + start_index - 1, - 0 - )`, - () => ch`indexOf(path_names, ${ch.param("String", endEventName ?? "")})`, - ); - const endIndex = branch( - hasEnd, - () => endIndexWithinStart, - () => ch`length(path_names)`, - ); - const prefixSteps = Math.ceil((definition.maxDepth - 1) / 2); - const suffixSteps = Math.floor((definition.maxDepth - 1) / 2); - const selectedPathWithoutStart = branch( - hasEnd, - () => ch`arraySlice( - compact_path, - greatest(1, end_index - ${ch.param("UInt8", definition.maxDepth)} + 1), - least(${ch.param("UInt8", definition.maxDepth)}, end_index) - )`, - () => ch`arraySlice(compact_path, 1, ${ch.param("UInt8", definition.maxDepth)})`, - ); - const selectedPathWithStart = branch( - hasEnd, - () => ch`if( - end_index - start_index + 1 <= ${ch.param("UInt8", definition.maxDepth)}, - arraySlice(compact_path, start_index, end_index - start_index + 1), - arrayConcat( - arraySlice(compact_path, start_index, ${ch.param("UInt8", prefixSteps)}), - [tuple( - '…', - tupleElement(compact_path[end_index - ${ch.param("UInt8", suffixSteps)} + 1], 2), - toInt64(0) - )], - arraySlice( - compact_path, - end_index - ${ch.param("UInt8", suffixSteps)} + 1, - ${ch.param("UInt8", suffixSteps)} - ) - ) - )`, - () => ch`arraySlice(compact_path, start_index, ${ch.param("UInt8", definition.maxDepth)})`, - ); - const selectedPath = branch( - hasStart, - () => selectedPathWithStart, - () => selectedPathWithoutStart, - ); - const endpointConditions = [ - ...branch( - hasStart, - () => [ch`start_index > 0`], - () => [], - ), - ...branch( - hasEnd, - () => [ch`end_index > 0`], - () => [], - ), - ...branch( - hasStart && hasEnd, - () => [ch`end_index >= start_index`], - () => [], - ), - ]; - const endpointWhere = branch( - endpointConditions.length > 0, - () => joinSqlFragments(ch, endpointConditions, " AND "), - () => ch`1 = 1`, - ); - const selectedPaths = ch` - SELECT - person_key, - ${selectedPath} AS selected_path - FROM ( - SELECT - person_key, - compact_path, - arrayMap(item -> item.1, compact_path) AS path_names, - ${startIndex} AS start_index, - ${endIndex} AS end_index - FROM (${compactedPaths}) - ) - WHERE ${endpointWhere} - `; - const edgeRows = ch` - SELECT - tupleElement(selected_path[target_step - 1], 1) AS source, - target_step - 1 AS source_step, - tupleElement(selected_path[target_step], 1) AS target, - target_step AS target_step, - dateDiff( - 'second', - tupleElement(selected_path[target_step - 1], 2), - tupleElement(selected_path[target_step], 2) - ) AS transition_seconds - FROM (${selectedPaths}) - ARRAY JOIN arrayEnumerate(selected_path) AS target_step - WHERE target_step > 1 - `; - const minEdgeCount = definition.minEdgeCount; - const maxEdgeCount = definition.maxEdgeCount; - const edgeConditions = [ - ...branch( - Boolean(minEdgeCount), - () => [ch`count() >= ${ch.param("UInt32", minEdgeCount)}`], - () => [], - ), - ...branch( - Boolean(maxEdgeCount), - () => [ch`count() <= ${ch.param("UInt32", maxEdgeCount)}`], - () => [], - ), - ]; - const edgeHaving = branch( - edgeConditions.length > 0, - () => ch`HAVING ${joinSqlFragments(ch, edgeConditions, " AND ")}`, - () => ch``, - ); - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - source, - source_step, - target, - target_step, - count() AS count, - avg(transition_seconds) AS average_transition_seconds - FROM (${edgeRows}) - GROUP BY source, source_step, target, target_step - ${edgeHaving} - ORDER BY count DESC, source_step ASC, source ASC, target ASC - LIMIT ${ch.param("UInt16", definition.edgeLimit ?? 50)} - `, - tenantSettings(input.organizationId), - ); - - return rows.map((row) => ({ - averageTransitionSeconds: Number(row.average_transition_seconds ?? 0), - count: Number(row.count ?? 0), - source: row.source, - sourceStep: Number(row.source_step), - target: row.target, - targetStep: Number(row.target_step), - })); - }); - -export interface EventStickinessQueryInput extends AnalyticsActorQueryInput { - readonly filters: CompiledAnalyticsFilter; - readonly interval: StickinessInsightQueryType["interval"]; - readonly occurrenceCriteria: NonNullable; - readonly organizationId: string; - readonly params: Pick; - readonly series: StickinessInsightQueryType["series"][number]; -} - -export interface EventStickinessBucket { - readonly count: number; - readonly intervals: number; -} - -interface EventStickinessRow { - active_intervals: string | number; - actor_count: string | number; -} - -/** Renders the per-interval occurrence predicate a stickiness bucket must satisfy. */ -const stickinessOccurrenceCondition = ( - ch: ClickhouseWebClient.ClickhouseWebClient, - criteria: EventStickinessQueryInput["occurrenceCriteria"], -): SqlFragment => { - switch (criteria.operator) { - case "exact": - return ch`count() = ${ch.param("UInt16", criteria.value)}`; - case "lte": - return ch`count() <= ${ch.param("UInt16", criteria.value)}`; - default: - return ch`count() >= ${ch.param("UInt16", criteria.value)}`; - } -}; - -/** Query the exact activity-frequency distribution for an identity-stitched event series. */ -export const getEventStickinessBuckets = (input: EventStickinessQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const periodExpression = getDateTruncExpression(EVENT_TS, input.interval); - const actorKey = analyticsActorKey(ch, input.actor); - const actorFilter = analyticsActorFilter(ch, input); - const propertyFilter = whenDefined( - input.series.filters, - (filters) => buildCustomEventFilter(ch, filters), - () => ch`1 = 1`, - ); - const occurrenceCondition = stickinessOccurrenceCondition(ch, input.occurrenceCriteria); - const dedupedEvents = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - AND event_ts >= ${ch.param("DateTime", toClickhouseDateTime(input.params.startDate))} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))} - AND event_name IN ${ch.param("Array(String)", [...input.series.eventNames])}`, - ); - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - active_intervals, - count() AS actor_count - FROM ( - SELECT - person_key, - count() AS active_intervals - FROM ( - SELECT - ${actorKey} AS person_key, - ${ch.literal(periodExpression)} AS active_period - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${propertyFilter} AND ${actorFilter} - GROUP BY person_key, active_period - HAVING ${occurrenceCondition} - ) - GROUP BY person_key - ) - GROUP BY active_intervals - ORDER BY active_intervals ASC - `, - tenantSettings(input.organizationId), - ); - - return rows.flatMap((row) => { - const intervals = Number(row.active_intervals); - const count = Number(row.actor_count); - if (!Number.isSafeInteger(intervals) || intervals <= 0 || !Number.isFinite(count)) return []; - return [{ count, intervals }]; - }); - }); - -export interface EventLifecycleQueryInput extends AnalyticsActorQueryInput { - readonly filters: CompiledAnalyticsFilter; - readonly granularity: LifecycleInsightQueryType["granularity"]; - readonly organizationId: string; - readonly params: Pick; - readonly series: LifecycleInsightQueryType["series"]; -} - -export interface EventLifecyclePoint { - readonly count: number; - readonly status: "dormant" | "new" | "resurrecting" | "returning"; - readonly timestamp: Date; -} - -interface EventLifecycleRow { - count: string | number; - period: string | Date; - status: EventLifecyclePoint["status"]; -} - -const startOfLifecycleInterval = ( - date: Date, - granularity: EventLifecycleQueryInput["granularity"], -): Date => { - const value = DateTime.makeUnsafe(date); - if (granularity === "hour") return DateTime.toDateUtc(DateTime.startOf(value, "hour")); - if (granularity === "day") return DateTime.toDateUtc(DateTime.startOf(value, "day")); - if (granularity === "week") { - // ISO weeks start on Monday, matching the previous `(getUTCDay() + 6) % 7` shift. - return DateTime.toDateUtc(DateTime.startOf(value, "week", { weekStartsOn: 1 })); - } - return DateTime.toDateUtc(DateTime.startOf(value, "month")); -}; - -/** The ClickHouse expression that advances a lifecycle period by one interval. */ -const addLifecycleIntervalExpression = ( - granularity: EventLifecycleQueryInput["granularity"], -): string => { - switch (granularity) { - case "hour": - return "addHours(current_period, 1)"; - case "day": - return "addDays(current_period, 1)"; - case "week": - return "addWeeks(current_period, 1)"; - default: - return "addMonths(current_period, 1)"; - } -}; - -/** Query new, returning, resurrecting, and dormant people for an event series. */ -export const getEventLifecyclePoints = (input: EventLifecycleQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const periodExpression = getDateTruncExpression(EVENT_TS, input.granularity); - const isGroupActor = input.actor?.kind === "group"; - const actorKey = branch( - isGroupActor, - () => analyticsActorKey(ch, input.actor), - () => ch`${ch.literal(effectivePersonIdExpression)}`, - ); - const actorFilter = branch( - isGroupActor, - () => analyticsActorFilter(ch, input), - () => ch`${analyticsActorFilter(ch, input)} - AND notEmpty(${ch.literal(effectivePersonIdExpression)})`, - ); - const propertyFilter = whenDefined( - input.series.filters, - (filters) => buildCustomEventFilter(ch, filters), - () => ch`1 = 1`, - ); - const matchingActivity = ch`( - ${ch.literal(`${EVENT_ALIAS}.event_name`)} IN ${ch.param("Array(String)", [ - ...input.series.eventNames, - ])} - AND ${propertyFilter} - )`; - const dedupedEvents = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", input.filters.projectIds)} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(input.params.endDate))}`, - ); - const actorActivity = ch` - SELECT - ${actorKey} AS person_key, - min(${ch.literal(periodExpression)}) AS first_seen_period, - arraySort( - groupUniqArrayIf(${ch.literal(periodExpression)}, ${matchingActivity}) - ) AS activity_periods - ${dedupedEvents} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - WHERE ${actorFilter} - GROUP BY person_key - HAVING notEmpty(activity_periods) - `; - const activeRows = ch` - SELECT - person_key, - activity_periods[activity_index] AS period, - multiIf( - period = first_seen_period, - 'new', - activity_index > 1 AND dateDiff( - ${ch.param("String", input.granularity)}, - activity_periods[activity_index - 1], - period - ) = 1, - 'returning', - 'resurrecting' - ) AS status - FROM (${actorActivity}) - ARRAY JOIN arrayEnumerate(activity_periods) AS activity_index - `; - const addIntervalExpression = addLifecycleIntervalExpression(input.granularity); - const dormantRows = ch` - SELECT - person_key, - ${ch.literal(addIntervalExpression)} AS period, - 'dormant' AS status - FROM ( - SELECT - person_key, - activity_periods, - activity_index, - activity_periods[activity_index] AS current_period - FROM (${actorActivity}) - ARRAY JOIN arrayEnumerate(activity_periods) AS activity_index - ) - WHERE activity_index = length(activity_periods) - OR dateDiff( - ${ch.param("String", input.granularity)}, - current_period, - activity_periods[activity_index + 1] - ) > 1 - `; - const rangeStart = startOfLifecycleInterval(input.params.startDate, input.granularity); - const rangeEnd = startOfLifecycleInterval(input.params.endDate, input.granularity); - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - period, - status, - uniqExact(person_key) AS count - FROM ( - ${activeRows} - UNION ALL - ${dormantRows} - ) - WHERE period >= ${ch.param("DateTime", toClickhouseDateTime(rangeStart))} - AND period <= ${ch.param("DateTime", toClickhouseDateTime(rangeEnd))} - GROUP BY period, status - ORDER BY period ASC, status ASC - `, - tenantSettings(input.organizationId), - ); - - return rows.flatMap((row) => { - const timestamp = parsePeriod(row.period); - if (!timestamp) return []; - return [{ count: Number(row.count ?? 0), status: row.status, timestamp }]; - }); - }); - -export interface AnalyticsQueryInput { - filters: CompiledAnalyticsFilter; - organizationId: string; - params: TimeRangeParams; -} - -export interface AnalyticsDataAccessor { - getRevenue: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getMRR: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getChurnedRevenue: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getActiveSubscriptions: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getActiveTrials: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getNewSubscriptions: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getChurnedSubscriptions: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getTrials: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getTrialConversions: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getPersonCount: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getNewPersons: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; - getPayingPersonCount: ( - input: AnalyticsQueryInput, - ) => Effect.Effect; -} - -const getRevenue = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: `coalesce(sum(${amountCentsExpression}), 0)`, - eventNames: ["$purchase.completed", "$subscription.created", "$subscription.renewed"], - }).pipe(Effect.map(moneyPoints)); - -const getMRR = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: `coalesce(sum(${amountCentsExpression}), 0)`, - eventNames: ["$subscription.created", "$subscription.renewed"], - extraWhere: `AND ${jsonBool("is_trial", "isTrial")} = 0`, - }).pipe(Effect.map(moneyPoints)); - -const getChurnedRevenue = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: `coalesce(sum(${amountCentsExpression}), 0)`, - eventNames: ["$subscription.canceled", "$subscription.expired"], - }).pipe(Effect.map(moneyPoints)); - -const getActiveSubscriptions = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: `countDistinct(${subscriptionIdExpression})`, - eventNames: ["$subscription.created", "$subscription.renewed", "$subscription.active"], - extraWhere: `AND ${jsonBool("is_trial", "isTrial")} = 0`, - }); - -const getActiveTrials = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: `countDistinct(${subscriptionIdExpression})`, - eventNames: ["$subscription.created", "$subscription.renewed", "$subscription.active"], - extraWhere: `AND ${jsonBool("is_trial", "isTrial")} = 1`, - }); - -const getNewSubscriptions = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: "count()", - eventNames: ["$subscription.created"], - extraWhere: `AND ${jsonBool("is_trial", "isTrial")} = 0`, - }); - -const getChurnedSubscriptions = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: "count()", - eventNames: ["$subscription.canceled", "$subscription.expired"], - }); - -const getTrials = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: "count()", - eventNames: ["$subscription.created"], - extraWhere: `AND ${jsonBool("is_trial", "isTrial")} = 1`, - }); - -const getTrialConversions = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: `countDistinct(${subscriptionIdExpression})`, - eventNames: ["$subscription.created", "$subscription.renewed", "$subscription.active"], - extraWhere: `AND ${jsonBool("converted_from_trial", "convertedFromTrial")} = 1`, - }); - -const getPersonCount = ({ params, filters, organizationId }: AnalyticsQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const period = getDateTruncExpression("first_seen", params.granularity); - const effectivePersonKey = `coalesce(${effectivePersonIdExpression}, ${effectiveDistinctIdExpression})`; - - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - ${ch.literal(period)} AS period, - count() AS total - FROM ( - SELECT - ${ch.literal(effectivePersonKey)} AS person_key, - min(${ch.literal(EVENT_TS)}) AS first_seen - ${resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", filters.projectIds)} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(params.endDate))}`, - )} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - GROUP BY person_key - ) - GROUP BY period - ORDER BY period ASC - `, - tenantSettings(organizationId), - ); - - return parseRowsToDataPoints(rows); - }); - -const getNewPersons = ({ params, filters, organizationId }: AnalyticsQueryInput) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const period = getDateTruncExpression("first_seen", params.granularity); - const effectivePersonKey = `coalesce(${effectivePersonIdExpression}, ${effectiveDistinctIdExpression})`; - - const rows = yield* ch.withClickhouseSettings( - ch` - SELECT - ${ch.literal(period)} AS period, - count() AS total - FROM ( - SELECT - ${ch.literal(effectivePersonKey)} AS person_key, - min(${ch.literal(EVENT_TS)}) AS first_seen - ${resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", filters.projectIds)} - AND event_ts <= ${ch.param("DateTime", toClickhouseDateTime(params.endDate))}`, - )} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - GROUP BY person_key - ) - WHERE first_seen >= ${ch.param("DateTime", toClickhouseDateTime(params.startDate))} - AND first_seen <= ${ch.param("DateTime", toClickhouseDateTime(params.endDate))} - GROUP BY period - ORDER BY period ASC - `, - tenantSettings(organizationId), - ); - - return parseRowsToDataPoints(rows); - }); - -const getPayingPersonCount = (input: AnalyticsQueryInput) => - eventMetric({ - ...input, - aggregateExpression: `countDistinct(${effectivePersonIdExpression})`, - eventNames: ["$purchase.completed", "$subscription.created", "$subscription.renewed"], - extraWhere: `AND ${amountCentsExpression} > 0`, - }); - -export interface ExperimentVariantResultRow { - readonly variant: string; - readonly exposures: number | string; - readonly conversions: number | string; - readonly revenue_cents: number | string; -} - -/** - * Per-variant experiment funnel: distinct exposed persons, distinct persons who - * fired a primary-metric event AFTER their first exposure, and post-exposure - * revenue (USD minor units). Exposures→conversions join on the identity-stitched - * person key (`coalesce(effectivePersonId, effectiveDistinctId)`) so an - * anonymous→identified user isn't undercounted, and events are deduped by - * `event_id` (latest-wins). Org-scoped by the readonly row policy. NOT a - * `BUILT_IN_INSIGHT` (those forbid the per-variant breakdown). - * - * Reads `$experiment.exposed` events (emitted server-side); returns no rows - * until exposure emission is producing events. NEEDS live-ClickHouse - * verification (value-level correctness cannot be checked by typecheck). - */ -export const getExperimentResults = (input: { - readonly organizationId: string; - readonly projectId: string; - readonly experimentId: string; - readonly primaryMetricEventNames: readonly string[]; - readonly revenueEventNames: readonly string[]; - readonly startDate: Date; - readonly endDate: Date; -}) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const personKey = `coalesce(${effectivePersonIdExpression}, ${effectiveDistinctIdExpression})`; - const variantExpr = `JSONExtractString(${EVENT_PROPERTIES}, 'variantKey')`; - const startTs = toClickhouseDateTime(input.startDate); - const endTs = toClickhouseDateTime(input.endDate); - const conversionEventNames = [ - ...new Set([...input.primaryMetricEventNames, ...input.revenueEventNames]), - ]; - - const exposuresFrom = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", [input.projectId])} - AND event_ts >= ${ch.param("DateTime", startTs)} - AND event_ts <= ${ch.param("DateTime", endTs)} - AND event_name = ${ch.param("String", "$experiment.exposed")} - AND JSONExtractString(event_properties, 'experimentId') = ${ch.param("String", input.experimentId)}`, - ); - const conversionsFrom = resolvedEventsFrom( - ch, - ch`project_id IN ${ch.param("Array(String)", [input.projectId])} - AND event_ts >= ${ch.param("DateTime", startTs)} - AND event_ts <= ${ch.param("DateTime", endTs)} - AND event_name IN ${ch.param("Array(String)", conversionEventNames)}`, - ); - - return yield* ch.withClickhouseSettings( - ch` - WITH - exposures AS ( - SELECT - ${ch.literal(personKey)} AS person_key, - argMin(${ch.literal(variantExpr)}, ${ch.literal(EVENT_TS)}) AS variant, - min(${ch.literal(EVENT_TS)}) AS first_ts - ${exposuresFrom} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - GROUP BY person_key - ), - conversions AS ( - SELECT - ${ch.literal(personKey)} AS person_key, - ${ch.literal(`${EVENT_ALIAS}.event_name`)} AS conv_event_name, - ${ch.literal(EVENT_TS)} AS conv_ts, - ${ch.literal(amountCentsExpression)} AS amount_cents - ${conversionsFrom} - ${ch.literal(RESOLVED_EVENTS_JOIN)} - ) - SELECT - exposures.variant AS variant, - countDistinct(exposures.person_key) AS exposures, - countDistinctIf( - conversions.person_key, - conversions.conv_ts >= exposures.first_ts - AND conversions.conv_event_name IN ${ch.param("Array(String)", [...input.primaryMetricEventNames])} - ) AS conversions, - coalesce(sumIf( - conversions.amount_cents, - conversions.conv_ts >= exposures.first_ts - AND conversions.conv_event_name IN ${ch.param("Array(String)", [...input.revenueEventNames])} - ), 0) AS revenue_cents - FROM exposures - LEFT JOIN conversions ON conversions.person_key = exposures.person_key - GROUP BY variant - ORDER BY variant ASC - `, - tenantSettings(input.organizationId), - ); - }); - -export const analyticsAccessor: AnalyticsDataAccessor = { - getActiveSubscriptions, - getActiveTrials, - getChurnedRevenue, - getChurnedSubscriptions, - getMRR, - getNewPersons, - getNewSubscriptions, - getPayingPersonCount, - getPersonCount, - getRevenue, - getTrialConversions, - getTrials, -}; - -export { - CLICKHOUSE_EVENTS_FULL_TABLE, - CLICKHOUSE_PERSONS_FULL_TABLE, - CLICKHOUSE_PENDING_OVERRIDES_FULL_TABLE, - // Reused verbatim by the VoidQL `events`/`revenue` logical-view lowering so the - // dedup + identity-resolution machinery is single-sourced (docs/analytics-access-layer.html §9). - RESOLVED_EVENTS_JOIN, - effectivePersonIdExpression, - effectiveDistinctIdExpression, - toClickhouseDateTime, -}; diff --git a/packages/core/src/services/analytics/postgres-series-resolver.ts b/packages/core/src/services/analytics/postgres-series-resolver.ts new file mode 100644 index 000000000..b1a6f7838 --- /dev/null +++ b/packages/core/src/services/analytics/postgres-series-resolver.ts @@ -0,0 +1,355 @@ +import type { + AnalyticsDataPoint, + BuiltInInsightId, + CompiledAnalyticsFilter, + TimeGranularity, +} from "../../domain/analytics/Analytics.ts"; +import { isReservedRevenueEventName } from "../../domain/internalAnalytics/InternalAnalyticsEvents.ts"; +import type { StoredAnalyticsEvent } from "./AnalyticsEventStore.ts"; +import { DateTime } from "effect"; + +const eventNames = { + subscriptionActivity: new Set([ + "$subscription.created", + "$subscription.renewed", + "$subscription.active", + ]), + subscriptionChurn: new Set(["$subscription.canceled", "$subscription.expired"]), +}; + +const property = (event: StoredAnalyticsEvent, ...keys: ReadonlyArray): unknown => { + for (const key of keys) { + const value = event.properties[key]; + if (value !== undefined && value !== null) return value; + } + return undefined; +}; + +const numberProperty = (event: StoredAnalyticsEvent, ...keys: ReadonlyArray): number => { + const value = property(event, ...keys); + if (typeof value !== "number" || !Number.isFinite(value)) return 0; + return value; +}; + +const booleanProperty = (event: StoredAnalyticsEvent, ...keys: ReadonlyArray): boolean => + keys.some((key) => event.properties[key] === true); + +const stringProperty = (event: StoredAnalyticsEvent, ...keys: ReadonlyArray): string => { + const value = property(event, ...keys); + if (typeof value !== "string") return ""; + return value; +}; + +const subscriptionId = (event: StoredAnalyticsEvent): string => + stringProperty( + event, + "subscription_id", + "subscriptionId", + "provider_subscription_id", + "providerSubscriptionId", + "store_subscription_id", + "storeSubscriptionId", + ) || event.eventId; + +const personKey = (event: StoredAnalyticsEvent): string => event.personId ?? event.distinctId; + +const dateFrom = (value: Date | number | string): Date => + DateTime.toDateUtc(DateTime.makeUnsafe(value)); + +const startOfBucket = (date: Date, granularity: TimeGranularity): Date => { + const result = dateFrom(date.getTime()); + result.setUTCMilliseconds(0); + result.setUTCSeconds(0); + result.setUTCMinutes(0); + if (granularity !== "hour") result.setUTCHours(0); + if (granularity === "week") { + const day = result.getUTCDay(); + let offset = day - 1; + if (day === 0) offset = 6; + result.setUTCDate(result.getUTCDate() - offset); + } + if (granularity === "month" || granularity === "quarter" || granularity === "year") { + result.setUTCDate(1); + } + if (granularity === "quarter") { + result.setUTCMonth(Math.floor(result.getUTCMonth() / 3) * 3); + } + if (granularity === "year") result.setUTCMonth(0); + return result; +}; + +const bucketKey = (event: StoredAnalyticsEvent, granularity: TimeGranularity): string => + startOfBucket(event.eventTimestamp, granularity).toISOString(); + +const withinRange = (event: StoredAnalyticsEvent, start: Date, end: Date): boolean => + event.eventTimestamp >= start && event.eventTimestamp <= end; + +const matchesFilters = (event: StoredAnalyticsEvent, filters: CompiledAnalyticsFilter): boolean => { + if (!filters.projectIds.includes(event.projectId)) return false; + const productId = stringProperty(event, "product_id", "productId", "product.id"); + if (filters.productIds?.length && !filters.productIds.includes(productId)) return false; + const environment = numberProperty(event, "provider_environment", "providerEnvironment"); + if (filters.providerEnvironments?.length && !filters.providerEnvironments.includes(environment)) { + return false; + } + const status = numberProperty(event, "subscription_status", "subscriptionStatus"); + if (filters.subscriptionStatuses?.length && !filters.subscriptionStatuses.includes(status)) { + return false; + } + return true; +}; + +const pointsFromValues = (values: ReadonlyMap): AnalyticsDataPoint[] => + [...values.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([timestamp, value]) => ({ timestamp: dateFrom(timestamp), value })); + +const sumByBucket = ( + events: ReadonlyArray, + granularity: TimeGranularity, + valueOf: (event: StoredAnalyticsEvent) => number, +): AnalyticsDataPoint[] => { + const values = new Map(); + for (const event of events) { + const key = bucketKey(event, granularity); + values.set(key, (values.get(key) ?? 0) + valueOf(event)); + } + return pointsFromValues(values); +}; + +const uniqueByBucket = ( + events: ReadonlyArray, + granularity: TimeGranularity, + keyOf: (event: StoredAnalyticsEvent) => string, +): AnalyticsDataPoint[] => { + const values = new Map>(); + for (const event of events) { + const key = bucketKey(event, granularity); + const bucket = values.get(key) ?? new Set(); + bucket.add(keyOf(event)); + values.set(key, bucket); + } + return pointsFromValues(new Map([...values].map(([key, bucket]) => [key, bucket.size]))); +}; + +const combine = ( + left: ReadonlyArray, + right: ReadonlyArray, + operation: (left: number, right: number) => number, +): AnalyticsDataPoint[] => { + const leftByTime = new Map(left.map((point) => [point.timestamp.toISOString(), point.value])); + const rightByTime = new Map(right.map((point) => [point.timestamp.toISOString(), point.value])); + const timestamps = new Set([...leftByTime.keys(), ...rightByTime.keys()]); + return [...timestamps] + .sort((a, b) => a.localeCompare(b)) + .map((timestamp) => ({ + timestamp: dateFrom(timestamp), + value: operation(leftByTime.get(timestamp) ?? 0, rightByTime.get(timestamp) ?? 0), + })); +}; + +const rate = (numerator: number, denominator: number): number => { + if (denominator <= 0) return 0; + return (numerator / denominator) * 100; +}; + +const ratio = (numerator: number, denominator: number): number => { + if (denominator <= 0) return 0; + return numerator / denominator; +}; + +const growthRate = (current: number, previous: number): number => { + if (previous !== 0) return rate(current - previous, previous); + if (current > 0) return 100; + return 0; +}; + +/** Computes one built-in analytics series from portable event rows. */ +export const resolvePostgresAnalyticsSeries = (input: { + readonly end: Date; + readonly events: ReadonlyArray; + readonly filters: CompiledAnalyticsFilter; + readonly granularity: TimeGranularity; + readonly insightId: BuiltInInsightId; + readonly start: Date; +}): AnalyticsDataPoint[] => { + const filtered = input.events.filter( + (event) => matchesFilters(event, input.filters) && withinRange(event, input.start, input.end), + ); + const cache = new Map(); + + const series = (insightId: BuiltInInsightId): AnalyticsDataPoint[] => { + const cached = cache.get(insightId); + if (cached) return cached; + let result: AnalyticsDataPoint[]; + switch (insightId) { + case "builtin/revenue": + result = sumByBucket( + filtered.filter((event) => isReservedRevenueEventName(event.eventName)), + input.granularity, + (event) => + numberProperty(event, "gross_amount_usd", "grossAmountUsd", "amount_usd", "amountUsd") / + 100, + ); + break; + case "builtin/mrr": + result = sumByBucket( + filtered.filter( + (event) => + (event.eventName === "$subscription.created" || + event.eventName === "$subscription.renewed") && + !booleanProperty(event, "is_trial", "isTrial"), + ), + input.granularity, + (event) => + numberProperty(event, "gross_amount_usd", "grossAmountUsd", "amount_usd", "amountUsd") / + 100, + ); + break; + case "builtin/arr": + result = series("builtin/mrr").map((point) => ({ ...point, value: point.value * 12 })); + break; + case "builtin/churned_revenue": + result = sumByBucket( + filtered.filter((event) => eventNames.subscriptionChurn.has(event.eventName)), + input.granularity, + (event) => + numberProperty(event, "gross_amount_usd", "grossAmountUsd", "amount_usd", "amountUsd") / + 100, + ); + break; + case "builtin/active_subscriptions": + case "builtin/active_trials": { + const trials = insightId === "builtin/active_trials"; + result = uniqueByBucket( + filtered.filter( + (event) => + eventNames.subscriptionActivity.has(event.eventName) && + booleanProperty(event, "is_trial", "isTrial") === trials, + ), + input.granularity, + subscriptionId, + ); + break; + } + case "builtin/new_subscriptions": + case "builtin/trials": { + const trials = insightId === "builtin/trials"; + result = sumByBucket( + filtered.filter( + (event) => + event.eventName === "$subscription.created" && + booleanProperty(event, "is_trial", "isTrial") === trials, + ), + input.granularity, + () => 1, + ); + break; + } + case "builtin/churned_subscriptions": + result = sumByBucket( + filtered.filter((event) => eventNames.subscriptionChurn.has(event.eventName)), + input.granularity, + () => 1, + ); + break; + case "builtin/trial_conversions": + result = uniqueByBucket( + filtered.filter( + (event) => + eventNames.subscriptionActivity.has(event.eventName) && + booleanProperty(event, "converted_from_trial", "convertedFromTrial"), + ), + input.granularity, + subscriptionId, + ); + break; + case "builtin/person_count": + case "builtin/new_persons": { + const firstSeen = new Map(); + for (const event of input.events.filter((candidate) => + matchesFilters(candidate, input.filters), + )) { + const key = personKey(event); + const existing = firstSeen.get(key); + if (!existing || existing.eventTimestamp > event.eventTimestamp) + firstSeen.set(key, event); + } + const firstSeenEvents = [...firstSeen.values()].filter((event) => { + if (insightId === "builtin/person_count") return event.eventTimestamp <= input.end; + return withinRange(event, input.start, input.end); + }); + result = sumByBucket(firstSeenEvents, input.granularity, () => 1); + break; + } + case "builtin/mrr_growth_rate": { + const mrr = series("builtin/mrr"); + result = mrr.map((point, index) => { + const previous = mrr[index - 1]?.value ?? 0; + const value = growthRate(point.value, previous); + return { timestamp: point.timestamp, value }; + }); + break; + } + case "builtin/churn_rate": + result = combine( + series("builtin/churned_subscriptions"), + series("builtin/active_subscriptions"), + (churned, active) => rate(churned, active + churned), + ); + break; + case "builtin/retention": + result = combine( + series("builtin/active_subscriptions"), + series("builtin/churned_subscriptions"), + (active, churned) => rate(active, active + churned), + ); + break; + case "builtin/arpu": + result = combine(series("builtin/revenue"), series("builtin/person_count"), ratio); + break; + case "builtin/arppu": { + const paying = uniqueByBucket( + filtered.filter( + (event) => + isReservedRevenueEventName(event.eventName) && + numberProperty( + event, + "gross_amount_usd", + "grossAmountUsd", + "amount_usd", + "amountUsd", + ) > 0, + ), + input.granularity, + personKey, + ); + result = combine(series("builtin/revenue"), paying, ratio); + break; + } + case "builtin/active_subscribers_growth": { + const active = series("builtin/active_subscriptions"); + result = active.map((point, index) => { + const previous = active[index - 1]?.value ?? 0; + return { + timestamp: point.timestamp, + value: growthRate(point.value, previous), + }; + }); + break; + } + case "builtin/subscriber_lifetime_value": + result = combine(series("builtin/arpu"), series("builtin/churn_rate"), (arpu, churn) => + ratio(arpu, churn / 100), + ); + break; + case "builtin/trial_conversion_rate": + result = combine(series("builtin/trial_conversions"), series("builtin/trials"), rate); + break; + } + cache.set(insightId, result); + return result; + }; + + return series(input.insightId); +}; diff --git a/packages/core/src/services/analytics/series-resolver.ts b/packages/core/src/services/analytics/series-resolver.ts deleted file mode 100644 index 20f53e3d7..000000000 --- a/packages/core/src/services/analytics/series-resolver.ts +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Pure series-resolution logic for the analytics insights operation. Splits - * the recursive metric-derivation graph (e.g. ARR = MRR x 12, churn-rate = - * churned / active+churned) out of the orchestration layer so the service - * file stays focused on permission checks and `catchTags`. - * - * `buildSeriesResolver` returns a closure with a per-call cache so derived - * metrics that share inputs (ARPU + ARPPU both use Revenue) don't re-issue - * the underlying ClickHouse query. Every metric is scoped to `organizationId`, - * which is passed through to the readonly ClickHouse user's tenant row - * policies and keyed into the cache. - */ -import { Effect, Schema } from "effect"; - -import type { - AnalyticsDataPoint, - BuiltInInsightId, - CompiledAnalyticsFilter, - TimeGranularity, - TimeRangeParams, -} from "../../domain/analytics/Analytics.ts"; -import type { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import type { SqlError } from "effect/unstable/sql/SqlError"; -import type { AnalyticsDataAccessor } from "./clickhouse-accessor.ts"; - -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); - -const calculateRate = (numerator: number, denominator: number): number => { - if (denominator > 0) return (numerator / denominator) * 100; - return 0; -}; - -const calculateGrowthRate = (current: number, previous: number): number => { - if (previous === 0) { - if (current > 0) return 100; - return 0; - } - return ((current - previous) / previous) * 100; -}; - -/** Divides two series values, treating a zero/missing denominator as `0`. */ -const safeDivide = (numerator: number, denominator: number | undefined): number => { - if (!denominator) return 0; - return numerator / denominator; -}; - -const executePrimitiveMetric = ({ - accessor, - compiledFilter, - insightId, - organizationId, - params, -}: { - accessor: AnalyticsDataAccessor; - compiledFilter: CompiledAnalyticsFilter; - insightId: BuiltInInsightId; - organizationId: string; - params: TimeRangeParams; -}): Effect.Effect => { - const input = { filters: compiledFilter, organizationId, params }; - switch (insightId) { - case "builtin/revenue": - return accessor.getRevenue(input); - case "builtin/mrr": - return accessor.getMRR(input); - case "builtin/churned_revenue": - return accessor.getChurnedRevenue(input); - case "builtin/active_subscriptions": - return accessor.getActiveSubscriptions(input); - case "builtin/active_trials": - return accessor.getActiveTrials(input); - case "builtin/new_subscriptions": - return accessor.getNewSubscriptions(input); - case "builtin/churned_subscriptions": - return accessor.getChurnedSubscriptions(input); - case "builtin/trials": - return accessor.getTrials(input); - case "builtin/trial_conversions": - return accessor.getTrialConversions(input); - case "builtin/person_count": - return accessor.getPersonCount(input); - case "builtin/new_persons": - return accessor.getNewPersons(input); - default: - return Effect.die(new Error(`Unexpected primitive insight ${insightId}`)); - } -}; - -export const buildSeriesResolver = (accessor: AnalyticsDataAccessor) => { - const cache = new Map(); - - const getSeries = ( - insightId: BuiltInInsightId, - compiledFilter: CompiledAnalyticsFilter, - granularity: TimeGranularity, - timeRange: { end: Date; start: Date }, - organizationId: string, - ): Effect.Effect => - Effect.gen(function* () { - const cacheKey = encodeJson({ - compiledFilter, - granularity, - insightId, - organizationId, - timeRange, - }); - const cached = cache.get(cacheKey); - if (cached) return cached; - - const params: TimeRangeParams = { - endDate: timeRange.end, - granularity, - startDate: timeRange.start, - }; - - const series = yield* Effect.gen(function* () { - switch (insightId) { - case "builtin/arr": { - const mrr = yield* getSeries( - "builtin/mrr", - compiledFilter, - granularity, - timeRange, - organizationId, - ); - return mrr.map((point) => ({ ...point, value: point.value * 12 })); - } - case "builtin/mrr_growth_rate": { - const mrr = yield* getSeries( - "builtin/mrr", - compiledFilter, - granularity, - timeRange, - organizationId, - ); - return mrr.map((point, index) => ({ - timestamp: point.timestamp, - value: calculateGrowthRate(point.value, mrr[index - 1]?.value ?? 0), - })); - } - case "builtin/churn_rate": { - const [churned, active] = yield* Effect.all([ - getSeries( - "builtin/churned_subscriptions", - compiledFilter, - granularity, - timeRange, - organizationId, - ), - getSeries( - "builtin/active_subscriptions", - compiledFilter, - granularity, - timeRange, - organizationId, - ), - ]); - return churned.map((point, index) => ({ - timestamp: point.timestamp, - value: calculateRate(point.value, (active[index]?.value ?? 0) + point.value), - })); - } - case "builtin/retention": { - const [active, churned] = yield* Effect.all([ - getSeries( - "builtin/active_subscriptions", - compiledFilter, - granularity, - timeRange, - organizationId, - ), - getSeries( - "builtin/churned_subscriptions", - compiledFilter, - granularity, - timeRange, - organizationId, - ), - ]); - return active.map((point, index) => ({ - timestamp: point.timestamp, - value: calculateRate(point.value, point.value + (churned[index]?.value ?? 0)), - })); - } - case "builtin/arpu": { - const [revenue, persons] = yield* Effect.all([ - getSeries("builtin/revenue", compiledFilter, granularity, timeRange, organizationId), - getSeries( - "builtin/person_count", - compiledFilter, - granularity, - timeRange, - organizationId, - ), - ]); - return revenue.map((point, index) => ({ - timestamp: point.timestamp, - value: safeDivide(point.value, persons[index]?.value), - })); - } - case "builtin/arppu": { - const [revenue, payingPersons] = yield* Effect.all([ - getSeries("builtin/revenue", compiledFilter, granularity, timeRange, organizationId), - accessor.getPayingPersonCount({ filters: compiledFilter, organizationId, params }), - ]); - return revenue.map((point, index) => ({ - timestamp: point.timestamp, - value: safeDivide(point.value, payingPersons[index]?.value), - })); - } - case "builtin/active_subscribers_growth": { - const activeSubscriptions = yield* getSeries( - "builtin/active_subscriptions", - compiledFilter, - granularity, - timeRange, - organizationId, - ); - return activeSubscriptions.map((point, index) => ({ - timestamp: point.timestamp, - value: calculateGrowthRate(point.value, activeSubscriptions[index - 1]?.value ?? 0), - })); - } - case "builtin/subscriber_lifetime_value": { - const [arpu, churnRate] = yield* Effect.all([ - getSeries("builtin/arpu", compiledFilter, granularity, timeRange, organizationId), - getSeries( - "builtin/churn_rate", - compiledFilter, - granularity, - timeRange, - organizationId, - ), - ]); - return arpu.map((point, index) => { - const churnPercentage = churnRate[index]?.value ?? 0; - if (churnPercentage <= 0) return { timestamp: point.timestamp, value: 0 }; - return { - timestamp: point.timestamp, - value: point.value / (churnPercentage / 100), - }; - }); - } - case "builtin/trial_conversion_rate": { - const [trials, conversions] = yield* Effect.all([ - getSeries("builtin/trials", compiledFilter, granularity, timeRange, organizationId), - getSeries( - "builtin/trial_conversions", - compiledFilter, - granularity, - timeRange, - organizationId, - ), - ]); - return trials.map((point, index) => ({ - timestamp: point.timestamp, - value: calculateRate(conversions[index]?.value ?? 0, point.value), - })); - } - default: - return yield* executePrimitiveMetric({ - accessor, - compiledFilter, - insightId, - organizationId, - params, - }); - } - }); - - cache.set(cacheKey, series); - return series; - }); - - return { getSeries }; -}; diff --git a/packages/core/src/services/analyticsIngest/AnalyticsDispatchService.ts b/packages/core/src/services/analyticsIngest/AnalyticsDispatchService.ts index 28c567db8..edae46e16 100644 --- a/packages/core/src/services/analyticsIngest/AnalyticsDispatchService.ts +++ b/packages/core/src/services/analyticsIngest/AnalyticsDispatchService.ts @@ -1,107 +1,47 @@ -/** - * `AnalyticsDispatchService` is the single seam both transports use to enqueue - * onto the shared analytics-ingest queue — a thin layer over the one - * {@link CaptureIngress} producer. - * - * - `dispatchCaptured` — the SDK path. Stamps each accepted envelope with an - * `Anonymous` / `Stitch` identity claim + `trustClass: "untrusted-sdk"`. - * - `dispatchTrusted` — the server-trusted revenue path. Maps each - * {@link InternalAnalyticsEvent} into a {@link CapturedEventV1} carrying a - * `Resolved` claim + `trustClass: "trusted-revenue"` + the trusted topic. - * - * Tests / non-worker hosts get {@link AnalyticsDispatchService.noop}. - */ -import { constant } from "@voidhash/lib/lang"; -import { Context, Effect, Layer } from "effect"; import type { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; +import { Context, DateTime, Effect, Layer } from "effect"; +import { analyticsEventFromInternal } from "../../domain/analytics/AnalyticsEvent.ts"; import { - type CapturedEventV1Type, - type CapturedIdentityClaim, - extractInnerProperties, - makeCapturedEventFromInternalAnalyticsEvent, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import type { InternalAnalyticsEvent } from "../../domain/internalAnalytics/InternalAnalyticsEvents.ts"; -import { - CaptureIngress, - type CaptureIngressError, - type PublishableCaptureEvent, -} from "./CaptureIngress.ts"; + isReservedRevenueEventName, + type InternalAnalyticsEvent, +} from "../../domain/internalAnalytics/InternalAnalyticsEvents.ts"; +import { AnalyticsEventStore } from "../analytics/AnalyticsEventStore.ts"; export interface AnalyticsDispatchServiceShape { - // `PlatformRuntime` marks the underlying queue send as runtime-only. `Db` - // (the ingest-DLQ write) is captured at the `CaptureIngress` layer's build, - // not per call, so it is not a method requirement — see `CaptureIngress`. - readonly dispatchCaptured: ( - events: ReadonlyArray, - ) => Effect.Effect; readonly dispatchTrusted: ( events: ReadonlyArray, - ) => Effect.Effect; + ) => Effect.Effect; } +const makeAnalyticsDispatchService = Effect.gen(function* () { + const store = yield* AnalyticsEventStore; + const dispatchTrusted = (events: ReadonlyArray) => + Effect.gen(function* () { + const processedAt = yield* DateTime.nowAsDate; + const revenueEvents = events + .filter((event) => isReservedRevenueEventName(event.eventName)) + .map((event) => analyticsEventFromInternal(event, processedAt)); + yield* store.insert(revenueEvents); + }); + + return { dispatchTrusted } satisfies AnalyticsDispatchServiceShape; +}); + /** - * Stamps the SDK identity claim + trust class onto an accepted capture - * envelope. `$identify` events with a `$previous_distinct_id` become a - * `Stitch`; everything else is `Anonymous`. Server-side stamping keeps the - * trust marker out of request-controlled input. + * Community trusted-event sink. Revenue events are synchronously upserted into + * PostgreSQL; other internal analytics classes are left to hosted editions. */ -export const stampSdkIdentityClaim = (envelope: CapturedEventV1Type): CapturedEventV1Type => { - const inner = extractInnerProperties(envelope.properties); - const identityClaim = (): CapturedIdentityClaim => { - const previousDistinctId = inner.$previous_distinct_id; - if ( - envelope.event === "$identify" && - typeof previousDistinctId === "string" && - previousDistinctId.length > 0 - ) { - return { _tag: "Stitch", distinctId: envelope.distinctId, previousDistinctId }; - } - return { _tag: "Anonymous", distinctId: envelope.distinctId }; - }; - return { ...envelope, identityClaim: identityClaim(), trustClass: "untrusted-sdk" }; -}; - export class AnalyticsDispatchService extends Context.Service< AnalyticsDispatchService, AnalyticsDispatchServiceShape ->()("@voidhash/core/AnalyticsDispatchService", { - make: Effect.gen(function* () { - const ingress = yield* CaptureIngress; - - const dispatchCaptured = (events: ReadonlyArray) => { - if (events.length === 0) return Effect.void; - return ingress.enqueueBatch( - events.map((event) => ({ - envelope: stampSdkIdentityClaim(event.envelope), - routeClass: event.routeClass, - })), - ); - }; - - const dispatchTrusted = (events: ReadonlyArray) => { - if (events.length === 0) return Effect.void; - return ingress.enqueueBatch( - events.map((event) => ({ - envelope: makeCapturedEventFromInternalAnalyticsEvent(event), - // Revenue is server-trusted and always lands on the main lane; the - // route never depends on quota (revenue never enters capture). - routeClass: constant("main"), - })), - ); - }; - - return { dispatchCaptured, dispatchTrusted } satisfies AnalyticsDispatchServiceShape; - }), -}) { - static readonly layer = Layer.effect(AnalyticsDispatchService)(AnalyticsDispatchService.make); +>()("@voidhash/core/AnalyticsDispatchService") { + static readonly layer: Layer.Layer = + Layer.effect(AnalyticsDispatchService)(makeAnalyticsDispatchService); - /** No-op dispatch for tests and non-worker hosts (mirrors {@link CaptureIngress.noop}). */ + /** No-op dispatch for tests and hosts that do not run analytics. */ static readonly noop: Layer.Layer = Layer.succeed( AnalyticsDispatchService, - { - dispatchCaptured: () => Effect.void, - dispatchTrusted: () => Effect.void, - }, + { dispatchTrusted: () => Effect.void }, ); } diff --git a/packages/core/src/services/analyticsIngest/AnalyticsIngestDlqService.ts b/packages/core/src/services/analyticsIngest/AnalyticsIngestDlqService.ts deleted file mode 100644 index ce0527674..000000000 --- a/packages/core/src/services/analyticsIngest/AnalyticsIngestDlqService.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { - analyticsIngestDlq, - AnalyticsIngestDlqReplayStatus, - Db, - desc, - eq, - sql, -} from "@voidhash/db"; -import { constant } from "@voidhash/lib/lang"; -import { Context, Effect, Layer, Schema } from "effect"; - -import type { RouteClass } from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import { CapturedEventV1 } from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import { generateId } from "../../utils/generate-id.ts"; -import { CaptureIngress } from "./CaptureIngress.ts"; - -export class AnalyticsIngestDlqServiceError extends Schema.TaggedErrorClass( - "AnalyticsIngestDlqServiceError", -)("AnalyticsIngestDlqServiceError", { cause: Schema.String }) {} - -export interface AnalyticsIngestDlqRecordFailureInput { - readonly attemptCount: number; - readonly captureId?: string; - readonly distinctId?: string; - readonly failureClass: string; - readonly failureMessage: string; - readonly payloadJson: unknown; - readonly projectId: string; - readonly routeClass: RouteClass; - readonly sourceSequence: number; - readonly sourceShard: string; -} - -const decodeCapturedEvent = Schema.decodeUnknownEffect(CapturedEventV1); -const decodeRouteClass = Schema.decodeUnknownEffect( - Schema.Literals(["main", "dlq", "overflow", "historical", "custom"]), -); - -export interface AnalyticsIngestDlqListInput { - readonly failureClass?: string; - readonly limit?: number; - readonly projectId?: string; -} - -export class AnalyticsIngestDlqService extends Context.Service()( - "AnalyticsIngestDlqService", - { - make: Effect.gen(function* () { - const db = yield* Db; - const recordFailure = Effect.fn("analyticsIngestDlq.recordFailure")(function* ( - input: AnalyticsIngestDlqRecordFailureInput, - ) { - const id = generateId("analyticsIngestDlq"); - yield* Effect.annotateCurrentSpan("voidhash.dlq.id", id); - yield* Effect.annotateCurrentSpan("voidhash.dlq.failure_class", input.failureClass); - yield* Effect.annotateCurrentSpan("voidhash.capture.route_class", input.routeClass); - if (input.projectId) - yield* Effect.annotateCurrentSpan("voidhash.project.id", input.projectId); - if (input.captureId) - yield* Effect.annotateCurrentSpan("voidhash.capture.id", input.captureId); - if (input.distinctId) - yield* Effect.annotateCurrentSpan("voidhash.person.distinct_id", input.distinctId); - yield* db - .insert(analyticsIngestDlq) - .values({ - attemptCount: input.attemptCount, - captureId: input.captureId, - distinctId: input.distinctId, - failureClass: input.failureClass, - failureMessage: input.failureMessage, - id, - payloadJson: input.payloadJson, - projectId: input.projectId, - routeClass: input.routeClass, - sourceSequence: input.sourceSequence, - sourceShard: input.sourceShard, - }) - .onConflictDoUpdate({ - target: analyticsIngestDlq.captureId, - set: { - attemptCount: input.attemptCount, - failureClass: input.failureClass, - failureMessage: input.failureMessage, - payloadJson: input.payloadJson, - replayStatus: AnalyticsIngestDlqReplayStatus.Pending, - updatedAt: sql`CURRENT_TIMESTAMP`, - }, - }); - return id; - }); - - const listFailures = Effect.fn("analyticsIngestDlq.listFailures")(function* ( - input: AnalyticsIngestDlqListInput = {}, - ) { - const limit = Math.max(1, Math.min(input.limit ?? 100, 500)); - if (input.projectId) - yield* Effect.annotateCurrentSpan("voidhash.project.id", input.projectId); - if (input.failureClass) - yield* Effect.annotateCurrentSpan("voidhash.dlq.failure_class", input.failureClass); - const filter = () => { - if (input.projectId && input.failureClass) - return sql`${analyticsIngestDlq.projectId} = ${input.projectId} AND ${analyticsIngestDlq.failureClass} = ${input.failureClass}`; - if (input.projectId) return eq(analyticsIngestDlq.projectId, input.projectId); - if (input.failureClass) return eq(analyticsIngestDlq.failureClass, input.failureClass); - return undefined; - }; - return yield* db - .select() - .from(analyticsIngestDlq) - .where(filter()) - .orderBy(desc(analyticsIngestDlq.createdAt)) - .limit(limit); - }); - - const markReplayed = Effect.fn("analyticsIngestDlq.markReplayed")(function* (id: string) { - yield* Effect.annotateCurrentSpan("voidhash.dlq.id", id); - yield* db - .update(analyticsIngestDlq) - .set({ - replayedAt: sql`CURRENT_TIMESTAMP`, - replayStatus: AnalyticsIngestDlqReplayStatus.Requeued, - }) - .where(eq(analyticsIngestDlq.id, id)); - }); - - const requeueFailure = Effect.fn("analyticsIngestDlq.requeueFailure")(function* (id: string) { - yield* Effect.annotateCurrentSpan("voidhash.dlq.id", id); - const ingress = yield* CaptureIngress; - const row = yield* db.query.analyticsIngestDlq.findFirst({ - where: { id }, - }); - if (!row) { - return yield* Effect.fail( - new AnalyticsIngestDlqServiceError({ cause: `DLQ row ${id} not found` }), - ); - } - if (row.projectId) yield* Effect.annotateCurrentSpan("voidhash.project.id", row.projectId); - if (row.routeClass) - yield* Effect.annotateCurrentSpan("voidhash.capture.route_class", row.routeClass); - const envelope = yield* decodeCapturedEvent(row.payloadJson).pipe( - Effect.mapError( - (cause) => - new AnalyticsIngestDlqServiceError({ - cause: `DLQ row ${id} payload is not a captured event: ${cause.message}`, - }), - ), - ); - const routeClass = yield* decodeRouteClass(row.routeClass).pipe( - Effect.mapError( - (cause) => - new AnalyticsIngestDlqServiceError({ - cause: `DLQ row ${id} has an unknown route class: ${cause.message}`, - }), - ), - ); - yield* ingress.enqueueBatch([{ envelope, routeClass }]); - yield* markReplayed(id); - }); - - return constant({ listFailures, markReplayed, recordFailure, requeueFailure }); - }), - }, -) { - static readonly layer = Layer.effect(AnalyticsIngestDlqService)(AnalyticsIngestDlqService.make); -} diff --git a/packages/core/src/services/analyticsIngest/AnalyticsJanitorService.ts b/packages/core/src/services/analyticsIngest/AnalyticsJanitorService.ts deleted file mode 100644 index c4991acdc..000000000 --- a/packages/core/src/services/analyticsIngest/AnalyticsJanitorService.ts +++ /dev/null @@ -1,387 +0,0 @@ -/** - * `AnalyticsJanitorService` reconciles pending identity merges in ClickHouse. - * One operation: `squash` selects a backlog of pending overrides older than the - * safety window, materialises a per-run shared staging table + a Dictionary over - * it, bulk `ALTER TABLE UPDATE`s the events table to assign the merged person - * ids, deletes the squashed backlog rows, then drops the staging resources. - * - * Transitive convergence: this squash takes the LATEST override version per - * `(project_id, source_distinct_id)` and applies it directly — no person-merge - * chain-following. That is correct because the identity merge keeps overrides - * canonical: when a person becomes non-canonical (an older person joins its - * component), the merge repoints that person's ENTIRE distinct-id cluster onto - * the new survivor (`PersonIdentityService.identifyDistinctId` → - * `IdentityMutationService.listMappedDistinctIds`), so the newest override for - * every distinct id already names the current canonical person. A transitive - * chain A→B→C therefore collapses in a single pass. - * - * ClickHouse Cloud constraint (why a staging MergeTree + Dictionary): on Cloud - * (`SharedMergeTree`) an `ALTER TABLE … UPDATE/DELETE` mutation runs - * ASYNCHRONOUSLY on other replicas, which cannot see session/node-local - * `ENGINE = Memory` / `ENGINE = Join` tables. So the per-run snapshot is a - * regular `MergeTree` (transparently `SharedMergeTree`, visible cluster-wide), - * and the per-row person-id lookup uses a `Dictionary` + `dictGet` rather than a - * node-local Join + `joinGet`. `dictGet` is non-deterministic, so the UPDATE - * needs `allow_nondeterministic_mutations`, passed as a session - * `clickhouse_setting` (Cloud ignores it in a SQL `SETTINGS` clause). - */ -import { constant } from "@voidhash/lib/lang"; -import { Clock, Context, DateTime, Effect, Layer, Schema } from "effect"; - -import { - type BacklogRow, - computeCutoffIso, - makeSnapshotResources, - type SnapshotResources, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; - -// Unqualified table names — the runtime Clickhouse client connects with the -// per-stage database (provisioned by `Clickhouse.Database`) as its default. -const CLICKHOUSE_EVENTS_FULL_TABLE = constant("events_v2"); -const CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_FULL_TABLE = constant( - "person_identity_pending_overrides_v2", -); - -export class AnalyticsJanitorServiceError extends Schema.TaggedErrorClass( - "AnalyticsJanitorServiceError", -)("AnalyticsJanitorServiceError", { - cause: Schema.String, - message: Schema.String, -}) {} - -export interface SquashInput { - readonly batchSize: number; - readonly safetyWindowSeconds: number; -} - -export interface SquashResult { - readonly backlogRowsProcessed: number; - readonly cutoffIso: string; - readonly durationMs: number; -} - -export class AnalyticsJanitorService extends Context.Service()( - "AnalyticsJanitorService", - { - make: Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - - const selectBacklog = ({ - batchSize, - cutoffIso, - }: { - readonly batchSize: number; - readonly cutoffIso: string; - }) => - ch`SELECT - project_id, - source_distinct_id, - target_distinct_id, - person_id, - version, - changed_at - FROM ( - SELECT - project_id, - source_distinct_id, - target_distinct_id, - person_id, - is_deleted, - version, - changed_at - FROM ${ch.literal(CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_FULL_TABLE)} - WHERE version > 0 - ORDER BY - project_id ASC, - source_distinct_id ASC, - version DESC, - changed_at DESC - LIMIT 1 BY project_id, source_distinct_id - ) - WHERE - is_deleted = 0 - AND changed_at < parseDateTime64BestEffort(${ch.param("String", cutoffIso)}) - ORDER BY changed_at ASC, project_id ASC, source_distinct_id ASC - LIMIT ${ch.param("UInt32", Math.max(0, Math.floor(batchSize)))}`; - - // Qualifies the per-run Dictionary (so an async mutation on another replica - // resolves it unambiguously) and points its `CLICKHOUSE` source at the - // staging table. The client's default is the per-stage analytics database. - const currentDatabaseName = ch<{ readonly db: string }>`SELECT currentDatabase() AS db`.pipe( - Effect.map((rows) => rows[0]?.db ?? ""), - ); - - // Per-run staging table — a regular `MergeTree` (transparently - // `SharedMergeTree` on Cloud) so it is visible cluster-wide to the async - // mutations below, unlike a node-local `ENGINE = Memory` table. - const createPendingOverrideSnapshot = (resources: SnapshotResources) => - ch.asCommand(ch`CREATE TABLE IF NOT EXISTS ${ch.literal(resources.pendingOverrideSnapshotName)} - ( - project_id String, - source_distinct_id String, - target_distinct_id String, - person_id String, - version UInt64, - changed_at DateTime64(3) - ) - ENGINE = MergeTree - ORDER BY (project_id, source_distinct_id)`); - - const insertPendingOverrideSnapshotRows = ({ - resources, - rows, - }: { - readonly resources: SnapshotResources; - readonly rows: ReadonlyArray; - }) => { - if (rows.length === 0) { - return Effect.void; - } - return ch - .insertQuery({ - table: resources.pendingOverrideSnapshotName, - values: rows, - }) - .pipe(Effect.asVoid); - }; - - // Escape a value for a single-quoted ClickHouse SQL string literal — the - // dictionary-source credentials live inside the `SOURCE(...)` clause and - // cannot be bound as query parameters. - const escapeChStringLiteral = (value: string): string => - value.replace(/\\/g, "\\\\").replace(/'/g, "\\'"); - - // Per-run Dictionary over the staging table, keyed by - // (project_id, source_distinct_id) → (person_id, version) — the ONLY per-run - // object the mutations reference, since a Dictionary is cluster-wide and - // resolves under an async mutation on any replica. Its `SOURCE` connects - // back as the bound (non-`default`) user, required to authenticate the - // `CLICKHOUSE` source on Cloud. `version` is carried so the backlog delete - // can scope by it. - const createPendingOverrideDictionary = ({ - databaseName, - resources, - }: { - readonly databaseName: string; - readonly resources: SnapshotResources; - }) => { - const dictionaryName = `${databaseName}.${resources.pendingOverrideDictionaryName}`; - const keyColumns = [ - { name: "project_id", type: "String" }, - { name: "source_distinct_id", type: "String" }, - ]; - const attributeColumns = [ - { name: "person_id", type: "String" }, - { name: "version", type: "UInt64" }, - ]; - const columns = [...keyColumns, ...attributeColumns] - .map((column) => `${column.name} ${column.type}`) - .join(", "); - const primaryKey = keyColumns.map((column) => column.name).join(", "); - const cfg = ch.config; - return ch.asCommand( - ch`${ch.literal(`CREATE DICTIONARY IF NOT EXISTS ${dictionaryName} (${columns}) - PRIMARY KEY ${primaryKey} - SOURCE(CLICKHOUSE( - USER '${escapeChStringLiteral(cfg.username ?? "")}' - PASSWORD '${escapeChStringLiteral(cfg.password ?? "")}' - DB '${escapeChStringLiteral(databaseName)}' - TABLE '${escapeChStringLiteral(resources.pendingOverrideSnapshotName)}' - )) - LAYOUT(COMPLEX_KEY_HASHED()) - LIFETIME(MIN 0 MAX 0)`)}`, - ); - }; - - // Passed as session `clickhouse_settings`, not a SQL `SETTINGS` clause: - // `dictGet`/`dictHas` are non-deterministic (so the mutation needs - // `allow_nondeterministic_mutations`), and Cloud only honours it there. - const mutationSettings = constant({ - mutations_sync: "1", - allow_nondeterministic_mutations: 1, - }); - - // Resolve merged person ids purely through the Dictionary (`dictHas` gates - // rows, `dictGet` supplies the id) so the async mutation never depends on - // resolving the per-run staging table on its replica. - const updatePersonIdsFromSnapshot = ({ - databaseName, - resources, - }: { - readonly databaseName: string; - readonly resources: SnapshotResources; - }) => { - const dictionary = `${databaseName}.${resources.pendingOverrideDictionaryName}`; - const lookup = `dictGet('${dictionary}', 'person_id', (project_id, distinct_id))`; - return ch.asCommand( - ch.withClickhouseSettings( - ch`${ch.literal(`ALTER TABLE ${CLICKHOUSE_EVENTS_FULL_TABLE} - UPDATE person_id = ${lookup} - WHERE dictHas('${dictionary}', (project_id, distinct_id)) - AND ( - person_id IS NULL - OR person_id != ${lookup} - )`)}`, - mutationSettings, - ), - ); - }; - - // Delete every pending-override version at-or-below the squashed version - // (the Dictionary's `version` attribute) for each key it covers. - const deleteSquashedBacklog = ({ - databaseName, - resources, - }: { - readonly databaseName: string; - readonly resources: SnapshotResources; - }) => { - const dictionary = `${databaseName}.${resources.pendingOverrideDictionaryName}`; - return ch.asCommand( - ch.withClickhouseSettings( - ch`${ch.literal(`ALTER TABLE ${CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_FULL_TABLE} - DELETE WHERE dictHas('${dictionary}', (project_id, source_distinct_id)) - AND version <= dictGetUInt64('${dictionary}', 'version', (project_id, source_distinct_id))`)}`, - mutationSettings, - ), - ); - }; - - const cleanupSnapshotResources = ({ - databaseName, - resources, - }: { - readonly databaseName: string; - readonly resources: SnapshotResources; - }) => - Effect.gen(function* () { - yield* ch - .asCommand( - ch`DROP DICTIONARY IF EXISTS ${ch.literal(`${databaseName}.${resources.pendingOverrideDictionaryName}`)}`, - ) - .pipe( - Effect.catchCause((cause) => - Effect.logError("failed to drop pending override dictionary", { - cause, - dictionary: resources.pendingOverrideDictionaryName, - }), - ), - ); - yield* ch - .asCommand( - ch`DROP TABLE IF EXISTS ${ch.literal(resources.pendingOverrideSnapshotName)}`, - ) - .pipe( - Effect.catchCause((cause) => - Effect.logError("failed to drop pending override snapshot", { - cause, - table: resources.pendingOverrideSnapshotName, - }), - ), - ); - }); - - const squash = Effect.fn("squash")( - function* (input: SquashInput) { - const startedAt = yield* Clock.currentTimeMillis; - const cutoffIso = computeCutoffIso({ - now: yield* DateTime.nowAsDate, - safetyWindowSeconds: input.safetyWindowSeconds, - }); - - yield* Effect.annotateCurrentSpan("voidhash.janitor.batch_size", input.batchSize); - yield* Effect.annotateCurrentSpan("voidhash.janitor.cutoff_iso", cutoffIso); - - const backlogRows = yield* selectBacklog({ - batchSize: input.batchSize, - cutoffIso, - }).pipe(Effect.withSpan("analytics-janitor.select-backlog")); - - yield* Effect.annotateCurrentSpan( - "voidhash.janitor.backlog_rows_processed", - backlogRows.length, - ); - - if (backlogRows.length === 0) { - const emptyDurationMs = (yield* Clock.currentTimeMillis) - startedAt; - yield* Effect.logInfo("analytics janitor found no eligible backlog rows", { - batchSize: input.batchSize, - cutoffIso, - durationMs: emptyDurationMs, - }); - return { - backlogRowsProcessed: 0, - cutoffIso, - durationMs: emptyDurationMs, - } satisfies SquashResult; - } - - const databaseName = yield* currentDatabaseName; - const snapshotResources = makeSnapshotResources(); - - yield* Effect.logInfo("analytics janitor selected backlog rows", { - count: backlogRows.length, - pendingOverrideDictionary: snapshotResources.pendingOverrideDictionaryName, - pendingOverrideSnapshot: snapshotResources.pendingOverrideSnapshotName, - cutoffIso, - }); - - yield* Effect.gen(function* () { - yield* createPendingOverrideSnapshot(snapshotResources).pipe( - Effect.withSpan("analytics-janitor.create-pending-override-snapshot"), - ); - yield* insertPendingOverrideSnapshotRows({ - resources: snapshotResources, - rows: backlogRows, - }).pipe(Effect.withSpan("analytics-janitor.insert-pending-override-snapshot-rows")); - yield* createPendingOverrideDictionary({ - databaseName, - resources: snapshotResources, - }).pipe(Effect.withSpan("analytics-janitor.create-pending-override-dictionary")); - yield* updatePersonIdsFromSnapshot({ - databaseName, - resources: snapshotResources, - }).pipe(Effect.withSpan("analytics-janitor.update-person-ids")); - yield* deleteSquashedBacklog({ - databaseName, - resources: snapshotResources, - }).pipe(Effect.withSpan("analytics-janitor.delete-backlog")); - }).pipe( - Effect.ensuring( - cleanupSnapshotResources({ databaseName, resources: snapshotResources }), - ), - ); - - const durationMs = (yield* Clock.currentTimeMillis) - startedAt; - yield* Effect.logInfo("analytics janitor completed squash run", { - count: backlogRows.length, - durationMs, - }); - - return { - backlogRowsProcessed: backlogRows.length, - cutoffIso, - durationMs, - } satisfies SquashResult; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - SqlError: (error) => - Effect.fail( - new AnalyticsJanitorServiceError({ - cause: error.message, - message: "analytics janitor squash failed", - }), - ), - }), - ), - ); - - return constant({ squash }); - }), - }, -) { - static readonly layer = Layer.effect(AnalyticsJanitorService)(AnalyticsJanitorService.make); -} diff --git a/packages/core/src/services/analyticsIngest/AnalyticsWriterService.ts b/packages/core/src/services/analyticsIngest/AnalyticsWriterService.ts deleted file mode 100644 index 73f039f7c..000000000 --- a/packages/core/src/services/analyticsIngest/AnalyticsWriterService.ts +++ /dev/null @@ -1,286 +0,0 @@ -/** - * `AnalyticsWriterService` consumes the three downstream wire events the - * processor emits and fans them out into five ClickHouse tables in parallel: - * `events_v2` (processed events), `persons_v1` (person profiles), - * `person_identity_v1` (identity mappings), `person_identity_overrides_v1` - * (override snapshots), and `person_identity_pending_overrides_v2` (merge queue). - * - * It yields {@link ClickhouseWebClient} directly (the Workers-safe driver) and - * uses Postgres only to resolve organization ids for person/identity rows. A - * runtime without ClickHouse acknowledges messages with zero inserted rows. - */ -import { Db } from "@voidhash/db"; -import { causeMessage, constant } from "@voidhash/lib/lang"; -import { Context, Effect, Layer, Option, Schema } from "effect"; - -import { - type AnalyticsWriterMessageType, - buildAnalyticsWriterPlan, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import { - isReservedRevenueEventName, - REVENUE_TRUSTED_SOURCE_TOPIC, -} from "../../domain/internalAnalytics/InternalAnalyticsEvents.ts"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; - -// Unqualified table names — the runtime Clickhouse client connects with the -// per-stage database (provisioned by `Clickhouse.Database`) as its default, -// so these resolve correctly without a hardcoded database prefix. -const CLICKHOUSE_EVENTS_FULL_TABLE = constant("events_v2"); -const CLICKHOUSE_PERSONS_FULL_TABLE = constant("persons_v1"); -const CLICKHOUSE_PERSON_IDENTITY_FULL_TABLE = constant("person_identity_v1"); -const CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_FULL_TABLE = constant("person_identity_overrides_v1"); -const CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_FULL_TABLE = constant( - "person_identity_pending_overrides_v2", -); - -export class AnalyticsWriterServiceError extends Schema.TaggedErrorClass( - "AnalyticsWriterServiceError", -)("AnalyticsWriterServiceError", { - cause: Schema.String, - message: Schema.String, -}) {} - -export interface WriteAnalyticsResult { - readonly insertedRowCount: number; - readonly messageCount: number; -} - -interface FetchExistingEventKeyRow { - project_id: string; - event_id: string; -} - -// Event dedup is scoped PER PROJECT and UNBOUNDED IN TIME: event_id derives from -// a stable id (the SDK client uuid, or the deterministic revenue id), so two -// tenants could send the same string. Keying on (project_id, event_id) stops one -// tenant's id from masking another's; length-prefixing the server-issued -// project_id keeps the key injective even if an event_id contains a separator. -// This is the SOLE dedup authority — `fetchExistingEventKeys` checks the whole -// batch against ClickHouse with no time bound, so a re-dispatched revenue event -// collapses no matter how late it arrives. -const eventDedupKey = (projectId: string, eventId: string): string => - `${projectId.length}:${projectId}${eventId}`; - -/** True for a trusted-revenue processed-event row (carries a deterministic `event_id`). */ -export const isRevenueAnalyticsWriterRow = (row: Readonly>): boolean => - row.source_topic === REVENUE_TRUSTED_SOURCE_TOPIC && - typeof row.event_name === "string" && - isReservedRevenueEventName(row.event_name); - -interface RevenueBatchDedupeResult { - readonly rows: ReadonlyArray>; - readonly skippedCount: number; -} - -/** - * Collapse trusted-revenue rows that appear twice in the SAME write batch (first - * seen wins, keyed on the deterministic `event_id`). Cross-batch and - * already-stored duplicates are caught by the unbounded `fetchExistingEventKeys` - * check; non-revenue rows pass through untouched. - */ -export const dedupeRevenueRowsWithinBatch = ( - rows: ReadonlyArray>, -): RevenueBatchDedupeResult => { - const seen = new Set(); - const deduped: Array> = []; - let skippedCount = 0; - - for (const row of rows) { - if (!isRevenueAnalyticsWriterRow(row) || typeof row.event_id !== "string") { - deduped.push(row); - continue; - } - const eventId = row.event_id; - if (seen.has(eventId)) { - skippedCount++; - continue; - } - seen.add(eventId); - deduped.push(row); - } - - if (skippedCount === 0) { - return { rows, skippedCount }; - } - return { rows: deduped, skippedCount }; -}; - -export class AnalyticsWriterService extends Context.Service()( - "AnalyticsWriterService", - { - make: Effect.gen(function* () { - const ch = Option.getOrUndefined( - yield* Effect.serviceOption(ClickhouseWebClient.ClickhouseWebClient), - ); - const db = yield* Db; - - // ClickHouse Cloud prefers large, infrequent inserts; the ingest consumer - // already folds a whole queue delivery into one `writeMessages` call, and - // `async_insert` lets the server coalesce parts server-side — the defence - // against the "too many parts" failure mode of small frequent inserts. - // `wait_for_async_insert: 1` keeps the write durable and backpressured: - // the insert resolves only once the server buffer has been flushed. - const insertRows = (table: string, rows: ReadonlyArray>) => { - if (rows.length === 0 || ch === undefined) { - return Effect.void; - } - return ch - .withClickhouseSettings(ch.insertQuery({ table, values: rows }), { - async_insert: 1, - wait_for_async_insert: 1, - }) - .pipe(Effect.asVoid); - }; - - const fetchExistingEventKeys = ( - projectIds: ReadonlyArray, - ids: ReadonlyArray, - ) => { - if (ids.length === 0 || ch === undefined) return Effect.succeed(new Set()); - return ch`SELECT project_id, event_id FROM ${ch.literal(CLICKHOUSE_EVENTS_FULL_TABLE)} - WHERE project_id IN ${ch.param("Array(String)", projectIds)} - AND event_id IN ${ch.param("Array(String)", ids)}`.pipe( - Effect.map( - (rows) => new Set(rows.map((row) => eventDedupKey(row.project_id, row.event_id))), - ), - ); - }; - - const writeMessages = Effect.fn("writeMessages")( - function* (messages: ReadonlyArray) { - yield* Effect.annotateCurrentSpan("voidhash.writer.message_count", messages.length); - if (ch === undefined) { - return { insertedRowCount: 0, messageCount: messages.length }; - } - let dedupSkippedTotal = 0; - - // Person / identity messages carry only `project_id`; resolve each to its - // `organization_id` (one MySQL lookup per batch over the distinct projects) - // so the written rows match the readonly user's tenant row policies. - // Processed events already carry their organization id. - const organizationProjectIds = [ - ...new Set( - messages - .filter((m) => m.kind === "person" || m.kind === "person-distinct-id") - .map((m) => m.value.projectId), - ), - ]; - let organizationByProject = new Map(); - if (organizationProjectIds.length > 0) { - const organizationRows = yield* db.query.projects.findMany({ - columns: { id: true, organizationId: true }, - where: { id: { in: organizationProjectIds } }, - }); - organizationByProject = new Map( - organizationRows.map((row) => [row.id, row.organizationId]), - ); - } - const plan = buildAnalyticsWriterPlan( - messages, - (projectId) => organizationByProject.get(projectId) ?? "", - ); - const batchDedupe = dedupeRevenueRowsWithinBatch(plan.processedEventRows); - let processedEventRows = batchDedupe.rows; - - const eventIds = [...new Set(processedEventRows.map((row) => String(row.event_id)))]; - const eventProjectIds = [ - ...new Set(processedEventRows.map((row) => String(row.project_id))), - ]; - const existingEventKeys = yield* fetchExistingEventKeys(eventProjectIds, eventIds); - if (existingEventKeys.size > 0) { - const beforeCount = processedEventRows.length; - processedEventRows = processedEventRows.filter( - (row) => - !existingEventKeys.has(eventDedupKey(String(row.project_id), String(row.event_id))), - ); - dedupSkippedTotal += beforeCount - processedEventRows.length; - yield* Effect.logInfo("skipped duplicate analytics events", { - eventDedupReason: "clickhouse_existing", - eventDedupSkippedCount: beforeCount - processedEventRows.length, - eventIdsInBatch: eventIds.length, - }); - } - - if (batchDedupe.skippedCount > 0) { - dedupSkippedTotal += batchDedupe.skippedCount; - yield* Effect.logInfo("skipped duplicate revenue analytics events", { - revenueDedupReason: "incoming_batch", - revenueDedupSkippedCount: batchDedupe.skippedCount, - }); - } - - yield* Effect.all( - [ - insertRows(CLICKHOUSE_EVENTS_FULL_TABLE, processedEventRows), - insertRows(CLICKHOUSE_PERSONS_FULL_TABLE, plan.personRows), - insertRows(CLICKHOUSE_PERSON_IDENTITY_FULL_TABLE, plan.personIdentityRows), - insertRows( - CLICKHOUSE_PERSON_IDENTITY_OVERRIDES_FULL_TABLE, - plan.personIdentityOverrideRows, - ), - insertRows( - CLICKHOUSE_PERSON_IDENTITY_PENDING_OVERRIDES_V2_FULL_TABLE, - plan.personIdentityPendingOverrideRows, - ), - ], - { concurrency: "unbounded" }, - ); - - const insertedRowCount = - processedEventRows.length + - plan.personRows.length + - plan.personIdentityRows.length + - plan.personIdentityOverrideRows.length + - plan.personIdentityPendingOverrideRows.length; - - yield* Effect.annotateCurrentSpan("voidhash.writer.inserted_row_count", insertedRowCount); - yield* Effect.annotateCurrentSpan( - "voidhash.writer.dedup_skipped_count", - dedupSkippedTotal, - ); - - return { - insertedRowCount, - messageCount: messages.length, - } satisfies WriteAnalyticsResult; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - SqlError: (error) => - Effect.fail( - new AnalyticsWriterServiceError({ - cause: error.message, - message: "failed to insert analytics messages", - }), - ), - EffectDrizzleQueryError: (error) => - Effect.fail( - new AnalyticsWriterServiceError({ - cause: causeMessage(error.cause ?? error.message), - message: "failed to resolve organization ids for analytics messages", - }), - ), - }), - ), - ); - - return constant({ writeMessages }); - }), - }, -) { - static readonly layer: Layer.Layer = Layer.effect( - AnalyticsWriterService, - )(AnalyticsWriterService.make); - - /** Builds the writer with an explicit read-write client instead of ambient analytics access. */ - static readonly layerWithClickhouse = ( - client: ClickhouseWebClient.ClickhouseWebClient, - ): Layer.Layer => - Layer.effect(AnalyticsWriterService)( - AnalyticsWriterService.make.pipe( - Effect.provideService(ClickhouseWebClient.ClickhouseWebClient, client), - ), - ); -} diff --git a/packages/core/src/services/analyticsIngest/CaptureIngress.ts b/packages/core/src/services/analyticsIngest/CaptureIngress.ts deleted file mode 100644 index 2e439f010..000000000 --- a/packages/core/src/services/analyticsIngest/CaptureIngress.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Context, Effect, Layer, Schema } from "effect"; -import type { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; - -import type { - CapturedEventV1Type, - RouteClass, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; - -export class CaptureIngressError extends Schema.TaggedErrorClass( - "CaptureIngressError", -)("CaptureIngressError", { - cause: Schema.optional(Schema.String), - message: Schema.String, -}) {} - -export interface CaptureIngressShape { - readonly enqueueBatch: ( - events: ReadonlyArray, - // `PlatformRuntime` marks the queue send as runtime-only. The ingest-DLQ - // write needs `Db`, but the live - // adapter captures it at the layer's build (it closes over the `Db`-in-make - // `AnalyticsIngestDlqService`), so it is no longer a per-call requirement. - ) => Effect.Effect; -} - -/** Accepted capture event plus the route selected by capture policy. */ -export interface PublishableCaptureEvent { - readonly envelope: CapturedEventV1Type; - readonly routeClass: RouteClass; -} - -/** - * Abstract ingress for accepted capture events. The queue-backed live adapter is - * wired at the application root, so `packages/core` carries no infrastructure - * dependency. - */ -export class CaptureIngress extends Context.Service()( - "@voidhash/core/CaptureIngress", -) { - static readonly noop: Layer.Layer = Layer.succeed(CaptureIngress, { - enqueueBatch: () => Effect.void, - }); -} diff --git a/packages/core/src/services/analyticsIngest/DlqProducer.ts b/packages/core/src/services/analyticsIngest/DlqProducer.ts deleted file mode 100644 index 843ccab11..000000000 --- a/packages/core/src/services/analyticsIngest/DlqProducer.ts +++ /dev/null @@ -1,88 +0,0 @@ -import type { Db } from "@voidhash/db"; -import { Context, Effect, Layer, Option, Schema } from "effect"; - -import type { - EventProcessorDlqV1, - RouteClass, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import { AnalyticsIngestDlqService } from "./AnalyticsIngestDlqService.ts"; - -export class DlqProducerError extends Schema.TaggedErrorClass("DlqProducerError")( - "DlqProducerError", - { - message: Schema.String, - cause: Schema.optional(Schema.String), - }, -) {} - -const decodeRawValue = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)); - -export interface DlqProducerShape { - readonly publishBatch: ( - events: ReadonlyArray, - ) => Effect.Effect; -} - -export class DlqProducer extends Context.Service()( - "@voidhash/core/DlqProducer", -) { - static readonly dbLive = Layer.effect( - DlqProducer, - Effect.gen(function* () { - const dlq = yield* AnalyticsIngestDlqService; - - const toRouteClass = (lane: EventProcessorDlqV1["lane"]): RouteClass => { - if (lane === "overflow" || lane === "historical") return lane; - return "main"; - }; - - const sourceSequence = (event: EventProcessorDlqV1): number => { - const suffix = event.sourceOffset.split(":").at(-1); - if (!suffix) return 0; - const parsed = Number.parseInt(suffix, 10); - if (!Number.isFinite(parsed)) return 0; - return parsed; - }; - - const payloadJson = (event: EventProcessorDlqV1): unknown => { - if (!event.rawValue) return event; - return Option.getOrElse(decodeRawValue(event.rawValue), () => event); - }; - - return { - publishBatch: (events) => - Effect.forEach( - events, - (event) => - dlq - .recordFailure({ - attemptCount: 0, - captureId: event.captureId, - distinctId: event.distinctId, - failureClass: event.failureClass, - failureMessage: event.failureMessage, - payloadJson: payloadJson(event), - projectId: event.projectId ?? "unknown", - routeClass: toRouteClass(event.lane), - sourceSequence: sourceSequence(event), - sourceShard: event.sourceTopic, - }) - .pipe( - Effect.mapError( - (error) => - new DlqProducerError({ - cause: String(error.cause), - message: "failed to record analytics ingest DLQ row", - }), - ), - ), - { discard: true }, - ), - } satisfies DlqProducerShape; - }), - ); - - static readonly noop: Layer.Layer = Layer.succeed(DlqProducer, { - publishBatch: () => Effect.void, - }); -} diff --git a/packages/core/src/services/analyticsIngest/EventCaptureService.ts b/packages/core/src/services/analyticsIngest/EventCaptureService.ts index c0004d9a3..c781b628c 100644 --- a/packages/core/src/services/analyticsIngest/EventCaptureService.ts +++ b/packages/core/src/services/analyticsIngest/EventCaptureService.ts @@ -1,39 +1,18 @@ -/** - * `EventCaptureService` orchestrates the capture pipeline: validate the inbound - * token, resolve the project + policy, enforce the request rate limit, then for - * each event enforce the per-event quota, pick a destination route, build the - * wire-stable {@link CapturedEventV1} envelope, and hand accepted envelopes to - * {@link CaptureIngress}. - * - * `CaptureRateLimitedError` / `CaptureUnauthorizedError` are part of the public - * HTTP contract and pass through as typed errors; every other infrastructural - * failure is wrapped as {@link EventCaptureServiceError} at the method boundary. - */ import { CaptureRateLimitedError, CaptureUnauthorizedError, type CaptureEvent, } from "@voidhash/api-contracts/event-capture"; -import { ANONYMOUS_USER_ID_PREFIX } from "@voidhash/lib"; -import { constant, pick } from "@voidhash/lib/lang"; +import { and, apiKeys, captureProjectPolicies, Db, eq, projects } from "@voidhash/db"; +import { constant } from "@voidhash/lib/lang"; +import type { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; import { Context, Effect, Layer, Schema } from "effect"; -import { createIdGenerator } from "../../utils/generate-id.ts"; - import { - type CaptureProjectPolicy, - type CapturedEventV1Type, - defaultCaptureProjectPolicy, - type RouteClass, - type RouteDecision, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import { and, apiKeys, captureProjectPolicies, Db, eq, projects } from "@voidhash/db"; -import { - isReservedRevenueEventName, - shouldBypassQuota, -} from "../../domain/internalAnalytics/InternalAnalyticsEvents.ts"; -import { CaptureIngress } from "./CaptureIngress.ts"; -import { PolicyCounterStore } from "./PolicyCounterStore.ts"; + analyticsEventFromCapture, + isCommunityCaptureEventName, +} from "../../domain/analytics/AnalyticsEvent.ts"; +import { AnalyticsEventStore } from "../analytics/AnalyticsEventStore.ts"; export class EventCaptureServiceError extends Schema.TaggedErrorClass( "EventCaptureServiceError", @@ -60,40 +39,19 @@ export interface CaptureResult { readonly rejected: number; } -interface ResolvedCaptureProject { - readonly organizationId: string; - readonly policy: CaptureProjectPolicy; - readonly projectId: string; +export interface EventCaptureServiceShape { + readonly captureEvents: ( + input: CaptureRequest, + ) => Effect.Effect< + CaptureResult, + EventCaptureServiceError | CaptureRateLimitedError | CaptureUnauthorizedError, + PlatformRuntime + >; } -const parseForceRoute = (value: string | null | undefined): RouteClass | undefined => { - if ( - value === "custom" || - value === "dlq" || - value === "historical" || - value === "main" || - value === "overflow" - ) { - return value; - } - return undefined; -}; - -/** Stable per-lane topic strings carried in the envelope's `routing.targetTopic`. */ -export const CAPTURE_TOPIC_MAIN = constant("capture.main.v1"); -export const CAPTURE_TOPIC_OVERFLOW = constant("capture.overflow.v1"); -export const CAPTURE_TOPIC_HISTORICAL = constant("capture.historical.v1"); -export const CAPTURE_TOPIC_DLQ = constant("capture.dlq.v1"); - -/** - * Mints the server-side capture id. Local prefix table (rather than the shared - * core one) because the id never leaves the capture envelope. - */ -const generateCaptureId = createIdGenerator(constant({ capture: "cap" })); - const TOKEN_FORMAT = /^vh_pk_\w+$/; -/** Validate and normalise (trim) an inbound publishable capture token. */ +/** Validate and normalize an inbound publishable capture token. */ export const validateCaptureToken = ( token: string, ): Effect.Effect => { @@ -111,376 +69,114 @@ export const validateCaptureToken = ( return Effect.succeed(normalized); }; -/** The last 4 chars of a token, used for redacted logging. */ +/** The last four token characters, safe for request diagnostics. */ export const tokenSuffix = (token: string): string => token.slice(-4); -/** Resolve the canonical event timestamp in priority order. */ -export const resolveEventTimestamp = ({ - receivedAt, - sentAt, - timestamp, -}: { - readonly sentAt?: Date; +/** Resolve the canonical event timestamp in capture priority order. */ +export const resolveEventTimestamp = (input: { readonly receivedAt: Date; + readonly sentAt?: Date; readonly timestamp?: Date; -}): Date => timestamp ?? sentAt ?? receivedAt; +}): Date => input.timestamp ?? input.sentAt ?? input.receivedAt; + +const makeEventCaptureService = Effect.gen(function* () { + const db = yield* Db; + const eventStore = yield* AnalyticsEventStore; + + const captureEvents = Effect.fn("captureEvents")( + function* (input: CaptureRequest) { + const token = yield* validateCaptureToken(input.request.token); + const [apiKeyRecord] = yield* db + .select({ + organizationId: projects.organizationId, + projectId: apiKeys.projectId, + }) + .from(apiKeys) + .innerJoin(projects, eq(projects.id, apiKeys.projectId)) + .where(and(eq(apiKeys.isPublic, true), eq(apiKeys.key, token))) + .limit(1); + + if (!apiKeyRecord) { + return yield* Effect.fail( + new CaptureUnauthorizedError({ code: "unauthorized", error: "invalid token" }), + ); + } -/** Pick the destination lane/topic for an accepted event from policy + quota state. */ -export const selectRoute = ({ - overQuota, - policy, -}: { - readonly overQuota: boolean; - readonly policy: CaptureProjectPolicy; -}): Effect.Effect => - Effect.gen(function* () { - const routeClass = policy.forceRoute ?? pick(overQuota, "overflow", "main"); + const [policy] = yield* db + .select({ ingestEnabled: captureProjectPolicies.ingestEnabled }) + .from(captureProjectPolicies) + .where(eq(captureProjectPolicies.projectId, apiKeyRecord.projectId)) + .limit(1); - switch (routeClass) { - case "main": - return { - isHistorical: false, - routeClass, - skipEnrichment: policy.skipEnrichment, - targetTopic: CAPTURE_TOPIC_MAIN, - }; - case "dlq": - return { - isHistorical: false, - routeClass, - skipEnrichment: policy.skipEnrichment, - targetTopic: CAPTURE_TOPIC_DLQ, - }; - case "overflow": - return { - isHistorical: false, - routeClass, - skipEnrichment: policy.skipEnrichment, - targetTopic: CAPTURE_TOPIC_OVERFLOW, - }; - case "historical": - return { - isHistorical: true, - routeClass, - skipEnrichment: policy.skipEnrichment, - targetTopic: CAPTURE_TOPIC_HISTORICAL, - }; - case "custom": - // Custom topics aren't wired in the Cloudflare-native infra; reject so the - // caller doesn't silently drop events on the floor. + if (policy?.ingestEnabled === false) { return yield* Effect.fail( new CaptureRateLimitedError({ code: "rate_limited", - error: "custom routes are not supported in this deployment", + error: "capture is disabled for this project", }), ); - } - }); - -/** - * Decide whether the event should materialise a person profile. - * - * An explicit attribute-set (`$set`/`$set_once` via the SDK's - * `setPersonAttributes`) is itself a reason to have a person, so the SDK stamps - * `$process_person_profile: true` even for anonymous distinct ids. That - * client-supplied boolean wins when present; otherwise only identified ids get a - * person profile. - */ -const resolveProcessPersonProfile = ({ - clientFlag, - distinctId, -}: { - readonly clientFlag: unknown; - readonly distinctId: string; -}): boolean => { - if (typeof clientFlag === "boolean") return clientFlag; - return !distinctId.startsWith(ANONYMOUS_USER_ID_PREFIX); -}; - -/** Map a stored capture-policy row (or its absence) onto the effective policy. */ -const toCaptureProjectPolicy = ( - record: typeof captureProjectPolicies.$inferSelect | undefined, - projectId: string, -): CaptureProjectPolicy => { - if (!record) return defaultCaptureProjectPolicy(projectId); - return { - customTopic: record.customTopic ?? undefined, - eventsPerDay: record.eventsPerDay ?? undefined, - forceRoute: parseForceRoute(record.forceRoute), - ingestEnabled: record.ingestEnabled, - projectId: record.projectId, - requestsPerMinute: record.requestsPerMinute ?? undefined, - skipEnrichment: record.skipEnrichment, - }; -}; - -/** Build the wire-stable {@link CapturedEventV1} envelope for an accepted event. */ -export const makeEnvelope = ({ - event, - organizationId, - projectId, - receivedAt, - request, - route, - sentAt, - token, -}: { - readonly event: typeof CaptureEvent.Type; - readonly organizationId: string; - readonly projectId: string; - readonly receivedAt: Date; - readonly request: { - readonly clientIp?: string; - readonly headers: Readonly>; - readonly path?: string; - readonly requestId: string; - }; - readonly route: RouteDecision; - readonly sentAt: Date; - readonly token: string; -}): CapturedEventV1Type => { - const timestamp = resolveEventTimestamp({ sentAt, receivedAt, timestamp: event.timestamp }); - const properties = { - distinctId: event.distinct_id, - properties: event.properties, - $process_person_profile: resolveProcessPersonProfile({ - clientFlag: event.properties.$process_person_profile, - distinctId: event.distinct_id, - }), - }; - const canonicalProperties = { - ...properties, - ...(typeof request.clientIp === "string" && { $ip: request.clientIp }), - }; + } + + const supported = input.events.filter((event) => isCommunityCaptureEventName(event.event)); + const records = supported.map((event) => + analyticsEventFromCapture({ + event, + organizationId: apiKeyRecord.organizationId, + projectId: apiKeyRecord.projectId, + receivedAt: input.request.receivedAt, + requestId: input.request.requestId, + requestPath: input.request.path, + sentAt: input.request.sentAt, + token, + }), + ); - return { - schemaVersion: 1, - captureId: generateCaptureId("capture"), - ...(event.uuid && { clientEventId: event.uuid }), - ...(event.session_id && { sessionId: event.session_id }), - context: event.context, - distinctId: event.distinct_id, - event: event.event, - eventTimestamp: timestamp.toISOString(), - organizationId, - projectId, - properties: canonicalProperties, - rawPayload: { - context: event.context, - distinct_id: event.distinct_id, - event: event.event, - properties, - ...(event.session_id && { session_id: event.session_id }), - ...(sentAt && { sent_at: sentAt }), - ...(event.timestamp && { timestamp: event.timestamp }), - ...(event.uuid && { uuid: event.uuid }), + yield* eventStore.insert(records); + const result = { + accepted: supported.length, + rejected: input.events.length - supported.length, + } satisfies CaptureResult; + + yield* Effect.logInfo("capture request processed", { + ...result, + projectId: apiKeyRecord.projectId, + requestId: input.request.requestId, + tokenSuffix: tokenSuffix(token), + }); + return result; }, - receivedAt: receivedAt.toISOString(), - request: { - requestId: request.requestId, - ...(request.path && { path: request.path }), - ...(request.headers["user-agent"] && { userAgent: request.headers["user-agent"] }), - ...(request.clientIp && { clientIp: request.clientIp }), - }, - routing: route, - token, - sentAt: sentAt.toISOString(), - }; -}; - -export class EventCaptureService extends Context.Service()( - "EventCaptureService", - { - make: Effect.gen(function* () { - const policyCounterStore = yield* PolicyCounterStore; - const ingress = yield* CaptureIngress; - const db = yield* Db; - - const captureEvents = Effect.fn("captureEvents")( - function* (input: CaptureRequest) { - const token = yield* validateCaptureToken(input.request.token); - - // Resolve the project + policy from the publishable token: look up the - // public api key (joined to its project), then load the project's - // capture policy (falling back to defaults), or fail unauthorized. - const [apiKeyRecord] = yield* db - .select({ - organizationId: projects.organizationId, - projectId: apiKeys.projectId, - }) - .from(apiKeys) - .innerJoin(projects, eq(projects.id, apiKeys.projectId)) - .where(and(eq(apiKeys.isPublic, true), eq(apiKeys.key, token))) - .limit(1); - - if (!apiKeyRecord) { - return yield* Effect.fail( - new CaptureUnauthorizedError({ code: "unauthorized", error: "invalid token" }), - ); - } - - const [policyRecord] = yield* db - .select() - .from(captureProjectPolicies) - .where(eq(captureProjectPolicies.projectId, apiKeyRecord.projectId)) - .limit(1); - - const project = { - organizationId: apiKeyRecord.organizationId, - policy: toCaptureProjectPolicy(policyRecord, apiKeyRecord.projectId), - projectId: apiKeyRecord.projectId, - } satisfies ResolvedCaptureProject; - - yield* Effect.annotateCurrentSpan("voidhash.request.id", input.request.requestId); - yield* Effect.annotateCurrentSpan("voidhash.api_key.suffix", tokenSuffix(token)); - yield* Effect.annotateCurrentSpan("voidhash.project.id", project.projectId); - if (project.organizationId) - yield* Effect.annotateCurrentSpan("voidhash.organization.id", project.organizationId); - - if (!project.policy.ingestEnabled) { - return yield* Effect.fail( - new CaptureRateLimitedError({ - code: "rate_limited", - error: "capture is disabled for this project", + (effect) => + effect.pipe( + Effect.withSpan("event-capture.captureEvents"), + Effect.catchTags({ + AnalyticsEventStoreError: (error) => + Effect.fail( + new EventCaptureServiceError({ cause: error.cause, message: error.message }), + ), + EffectDrizzleQueryError: (error) => + Effect.fail( + new EventCaptureServiceError({ + cause: String(error.cause), + message: "capture project lookup failed", }), - ); - } + ), + }), + ), + ); - const requestLimit = yield* policyCounterStore.checkRequestLimit({ - now: input.request.receivedAt, - projectId: project.projectId, - requestsPerMinute: project.policy.requestsPerMinute, - }); - if (!requestLimit.allowed) { - return yield* Effect.fail( - new CaptureRateLimitedError({ - code: "rate_limited", - error: "request rate limit exceeded", - ...(typeof requestLimit.retryAfterMs === "number" && { - retry_after_ms: requestLimit.retryAfterMs, - }), - }), - ); - } - - const publishableEvents: Array<{ - envelope: ReturnType; - routeClass: RouteClass; - }> = []; - let accepted = 0; - let rejected = 0; - - for (const event of input.events) { - if (isReservedRevenueEventName(event.event)) { - rejected += 1; - yield* Effect.logWarning( - "rejected reserved revenue event from publishable-key capture", - { - eventName: event.event, - projectId: project.projectId, - tokenSuffix: tokenSuffix(token), - }, - ); - continue; - } - - const outcome = yield* Effect.result( - Effect.gen(function* () { - // The seam to exempt specific event classes from quota. When - // bypassed, the counter is never read and the route is never - // forced to overflow; SDK events consume quota by default. - const bypassQuota = shouldBypassQuota({ - eventName: event.event, - trustClass: "untrusted-sdk", - }); - const withinQuota = - bypassQuota || - (yield* policyCounterStore - .checkEventQuota({ - now: input.request.receivedAt, - projectId: project.projectId, - quota: project.policy.eventsPerDay, - }) - .pipe(Effect.withSpan("policy.apply"))); - - const route = yield* selectRoute({ - overQuota: !withinQuota, - policy: project.policy, - }); - - const envelope = makeEnvelope({ - event, - organizationId: project.organizationId, - projectId: project.projectId, - receivedAt: input.request.receivedAt, - request: input.request, - route, - sentAt: input.request.sentAt, - token, - }); - - return { envelope, routeClass: route.routeClass }; - }), - ); + return constant({ captureEvents }) satisfies EventCaptureServiceShape; +}); - if (outcome._tag === "Failure") { - rejected += 1; - continue; - } - - publishableEvents.push(outcome.success); - accepted += 1; - } - - yield* ingress.enqueueBatch(publishableEvents); - yield* Effect.annotateCurrentSpan("voidhash.capture.accepted_count", accepted); - yield* Effect.annotateCurrentSpan("voidhash.capture.rejected_count", rejected); - yield* Effect.logInfo("capture request processed", { - accepted, - projectId: project.projectId, - rejected, - requestId: input.request.requestId, - tokenSuffix: tokenSuffix(token), - }); - - return { accepted, rejected } satisfies CaptureResult; - }, - (effect) => - effect.pipe( - Effect.withSpan("event-capture.captureEvents"), - Effect.catchTags({ - EffectDrizzleQueryError: (error) => - Effect.fail( - new EventCaptureServiceError({ - cause: String(error.cause), - message: "capture project lookup failed", - }), - ), - PolicyStoreError: (error) => - Effect.fail( - new EventCaptureServiceError({ - cause: String(error.cause ?? error.message), - message: error.message, - }), - ), - CaptureIngressError: (error) => - Effect.fail( - new EventCaptureServiceError({ - cause: String(error.cause ?? error.message), - message: error.message, - }), - ), - }), - ), - ); - - return constant({ captureEvents }); - }), - }, -) { - static readonly layer: Layer.Layer< - EventCaptureService, - never, - Db | PolicyCounterStore | CaptureIngress - > = Layer.effect(EventCaptureService)(EventCaptureService.make); +/** + * Community capture implementation. It accepts only the built-in lifecycle + * events and inserts them synchronously into PostgreSQL. Unsupported and + * custom events are intentionally counted as rejected without entering a + * queue, workflow, identity processor, or dead-letter path. + */ +export class EventCaptureService extends Context.Service< + EventCaptureService, + EventCaptureServiceShape +>()("EventCaptureService") { + static readonly layer: Layer.Layer = + Layer.effect(EventCaptureService)(makeEventCaptureService); } diff --git a/packages/core/src/services/analyticsIngest/EventProcessorService.ts b/packages/core/src/services/analyticsIngest/EventProcessorService.ts deleted file mode 100644 index ca03a5d94..000000000 --- a/packages/core/src/services/analyticsIngest/EventProcessorService.ts +++ /dev/null @@ -1,656 +0,0 @@ -/** - * `EventProcessorService` turns one captured record into the downstream wire - * events the writer consumes: resolve the project + processor policy (DLQ on - * failure), validate policy/lane preconditions via {@link attachProjectPolicy} - * (DLQ on rejection), resolve identity through {@link PersonIdentityService}, - * build the wire-stable {@link ProcessedEventV2}, and return the three output - * streams. - * - * Records for the same `(token, distinctId)` are safe to process in any order: - * identity resolution is order-agnostic (oldest-wins union-find + per-trait LWW) - * and serialized at the row level by the identity transaction's `FOR UPDATE` - * locks — no application-level scheduler needed. - */ -import { Context, DateTime, Effect, Layer, Schema } from "effect"; - -import { ANONYMOUS_USER_ID_PREFIX } from "@voidhash/lib"; -import { constant, pick } from "@voidhash/lib/lang"; -import { and, apiKeys, captureProjectPolicies, Db, eq, projects } from "@voidhash/db"; - -import { - buildDlqEvent, - type CapturedEventV1Type, - type CapturedTransportRecord, - type EventProcessorDlqV1, - extractInnerProperties, - parsePersonTraits, - type ProcessedEventIdentity, - type ProcessedEventV2Type, - type ProcessingEvent, - type ProcessorLane, - type ProcessorPersonEventV1Type, - type ProcessorPersonIdentityEventV1Type, - type ProcessorProjectPolicy, - type ResolvedProcessorProject, - validateBuiltInProcessorRules, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; -import { - isReservedRevenueEventName, - REVENUE_TRUSTED_SOURCE_TOPIC, -} from "../../domain/internalAnalytics/InternalAnalyticsEvents.ts"; -import { - PersonIdentityService, - type PersonIdentityEventV1, - type PersonIdentityResult, - type PersonSnapshotEventV1, - type ResolvedAnalyticsIdentity, -} from "../personIdentity/PersonIdentityService.ts"; -import { DlqProducer } from "./DlqProducer.ts"; -import type { ProcessorOutputs } from "./ProcessorOutputs.ts"; - -export class EventProcessorServiceError extends Schema.TaggedErrorClass( - "EventProcessorServiceError", -)("EventProcessorServiceError", { - cause: Schema.String, - message: Schema.String, -}) {} - -const DEFAULT_PROCESSOR_POLICY: ProcessorProjectPolicy = { - processorAllowHistorical: true, - processorAllowOverflow: true, - processorEnabled: true, - processorHistoricalMinAgeHours: 48, - processorPersonProcessingEnabled: true, - processorSchemaMode: "reject", -}; - -const buildProcessedEventIdentity = ( - identity: ResolvedAnalyticsIdentity, -): ProcessedEventIdentity => { - const optional: { personId?: string } = {}; - if (identity.personId) optional.personId = identity.personId; - return { - ...optional, - distinctId: identity.distinctId, - mode: identity.mode, - }; -}; - -/** Builds the wire-stable {@link ProcessedEventV2} the writer consumes. */ -export const buildProcessedEvent = ({ - capturedEvent, - identity, - lane, - sourceOffset, - sourcePartition, - sourceTopic, -}: { - readonly capturedEvent: CapturedEventV1Type; - readonly identity: ResolvedAnalyticsIdentity; - readonly lane: ProcessorLane; - readonly sourceOffset: string; - readonly sourcePartition: number; - readonly sourceTopic: string; -}): ProcessedEventV2Type => { - const optional: { sessionId?: string } = {}; - if (capturedEvent.sessionId) optional.sessionId = capturedEvent.sessionId; - return { - captureId: capturedEvent.captureId, - context: capturedEvent.context, - distinctId: capturedEvent.distinctId, - event: capturedEvent.event, - eventTimestamp: capturedEvent.eventTimestamp, - groups: [], - identity: buildProcessedEventIdentity(identity), - organizationId: capturedEvent.organizationId, - processedAt: DateTime.formatIso(DateTime.nowUnsafe()), - // The ClickHouse dedup key prefers the SDK's stable client uuid (reused across - // retries / offline redelivery) over the fresh per-request captureId, so an - // SDK-level resend collapses on read; falls back to captureId for non-SDK callers. - processedEventId: capturedEvent.clientEventId ?? capturedEvent.captureId, - projectId: capturedEvent.projectId, - properties: capturedEvent.properties, - request: capturedEvent.request, - routing: { - lane, - skipEnrichment: capturedEvent.routing.skipEnrichment, - sourceOffset, - sourcePartition, - sourceTopic, - }, - schemaVersion: 2, - ...optional, - token: capturedEvent.token, - }; -}; - -/** - * Validate policy + lane preconditions for a captured record and, on success, - * promote it to a {@link ProcessingEvent}; on rejection return the pre-built - * {@link EventProcessorDlqV1} for the caller to publish. - */ -export const attachProjectPolicy = ({ - now, - record, - resolvedProject, -}: { - readonly now: Date; - readonly record: CapturedTransportRecord; - readonly resolvedProject: ResolvedProcessorProject; -}): - | { readonly ok: true; readonly value: ProcessingEvent } - | { readonly ok: false; readonly value: EventProcessorDlqV1 } => { - const reject = ( - failureClass: EventProcessorDlqV1["failureClass"], - failureMessage: string, - ): { readonly ok: false; readonly value: EventProcessorDlqV1 } => ({ - ok: false, - value: buildDlqEvent({ - captureId: record.capturedEvent.captureId, - distinctId: record.capturedEvent.distinctId, - failureClass, - failureMessage, - headers: record.headers, - lane: record.lane, - projectId: record.capturedEvent.projectId, - rawKey: record.rawKey, - rawValue: record.rawValue, - sourceOffset: record.sourceOffset, - sourcePartition: record.sourcePartition, - sourceTopic: record.sourceTopic, - token: record.capturedEvent.token, - }), - }); - - if (resolvedProject.projectId !== record.capturedEvent.projectId) { - return reject("project_not_found", "captured event project id does not match resolved token"); - } - if (!resolvedProject.policy.processorEnabled) { - return reject("policy_rejected", "processor is disabled for the project"); - } - if (record.lane === "overflow" && !resolvedProject.policy.processorAllowOverflow) { - return reject("policy_rejected", "overflow lane is disabled for the project"); - } - if (record.lane === "historical" && !resolvedProject.policy.processorAllowHistorical) { - return reject("policy_rejected", "historical lane is disabled for the project"); - } - - const validationError = validateBuiltInProcessorRules({ - capturedEvent: record.capturedEvent, - historicalMinAgeHours: resolvedProject.policy.processorHistoricalMinAgeHours, - lane: record.lane, - now, - sourceTopic: record.sourceTopic, - }); - if (validationError) return reject("schema_rejected", validationError); - - return { - ok: true, - value: { - capturedEvent: record.capturedEvent, - headers: record.headers, - identityKey: `${record.capturedEvent.token}:${record.capturedEvent.distinctId}`, - lane: record.lane, - projectPolicy: resolvedProject.policy, - rawKey: record.rawKey, - rawValue: record.rawValue, - sourceOffset: record.sourceOffset, - sourcePartition: record.sourcePartition, - sourceTopic: record.sourceTopic, - }, - }; -}; - -const parseProcessPersonProfile = ( - properties: Record, -): Effect.Effect => - Effect.gen(function* () { - const rawValue = properties.$process_person_profile; - if (typeof rawValue === "undefined") return undefined; - if (typeof rawValue !== "boolean") { - return yield* new EventProcessorServiceError({ - cause: "invalid_process_person_profile", - message: "$process_person_profile must be a boolean", - }); - } - return rawValue; - }); - -const parseIdentifySourceDistinctId = ( - properties: Record, -): Effect.Effect => - Effect.gen(function* () { - const rawValue = properties.$previous_distinct_id; - if (typeof rawValue !== "string" || rawValue.length === 0) { - return yield* new EventProcessorServiceError({ - cause: "missing_previous_distinct_id", - message: "$identify requires properties.$previous_distinct_id", - }); - } - return rawValue; - }); - -/** Returns the first candidate that is a string — the trait `set`/`setOnce` fallback chain. */ -const firstString = (...candidates: ReadonlyArray): string | undefined => { - for (const candidate of candidates) { - if (typeof candidate === "string") return candidate; - } - return undefined; -}; - -/** The {@link PersonIdentityService} call computed for a processing event. */ -export type PersonIdentityCall = - | { - readonly kind: "identify"; - readonly input: { - readonly distinctId: string; - readonly email?: string; - readonly eventId: string; - readonly eventTimestamp: Date; - readonly name?: string; - readonly previousDistinctId: string; - readonly projectId: string; - readonly setAttributes: Record; - readonly setOnceAttributes: Record; - }; - } - | { - readonly kind: "resolve"; - readonly input: { - readonly distinctId: string; - readonly email?: string; - readonly eventId: string; - readonly eventTimestamp: Date; - readonly name?: string; - readonly projectId: string; - readonly setAttributes: Record; - readonly setOnceAttributes: Record; - readonly shouldCreatePerson: boolean; - }; - }; - -/** - * Computes the {@link PersonIdentityService} call (identify vs resolve) for a - * processing event, extracting person traits and the stable identity `eventId` - * (the SDK client uuid, falling back to captureId) used for assertion dedup. - * - * Fails with {@link EventProcessorServiceError} when the event carries - * malformed person traits; the processor treats that as a defect. - */ -export const buildPersonIdentityCall = ( - processingEvent: ProcessingEvent, -): Effect.Effect => - Effect.gen(function* () { - const { capturedEvent } = processingEvent; - const innerProperties = extractInnerProperties(capturedEvent.properties); - const traits = parsePersonTraits(innerProperties); - if (!traits.ok) - return yield* new EventProcessorServiceError({ - cause: "invalid_person_traits", - message: traits.message, - }); - - const name = firstString(traits.value.set.name, traits.value.setOnce.name); - const email = firstString(traits.value.set.email, traits.value.setOnce.email); - const setAttributes = Object.fromEntries( - Object.entries(traits.value.set).filter(([key]) => key !== "email" && key !== "name"), - ); - const setOnceAttributes = Object.fromEntries( - Object.entries(traits.value.setOnce).filter(([key]) => key !== "email" && key !== "name"), - ); - - const eventId = capturedEvent.clientEventId ?? capturedEvent.captureId; - - if (capturedEvent.event === "$identify") { - const previousDistinctId = yield* parseIdentifySourceDistinctId(innerProperties); - return { - kind: "identify", - input: { - distinctId: capturedEvent.distinctId, - email, - eventId, - eventTimestamp: DateTime.toDateUtc(DateTime.makeUnsafe(capturedEvent.eventTimestamp)), - name, - previousDistinctId, - projectId: capturedEvent.projectId, - setAttributes, - setOnceAttributes, - }, - }; - } - - const processPersonProfile = yield* parseProcessPersonProfile(capturedEvent.properties); - const shouldCreatePerson = - processPersonProfile ?? !capturedEvent.distinctId.startsWith(ANONYMOUS_USER_ID_PREFIX); - - const enrichmentDisabled = - capturedEvent.routing.skipEnrichment || - !processingEvent.projectPolicy.processorPersonProcessingEnabled; - - return { - kind: "resolve", - input: { - distinctId: capturedEvent.distinctId, - email, - eventId, - eventTimestamp: DateTime.toDateUtc(DateTime.makeUnsafe(capturedEvent.eventTimestamp)), - name, - projectId: capturedEvent.projectId, - setAttributes: pick(enrichmentDisabled, {}, setAttributes), - setOnceAttributes: pick(enrichmentDisabled, {}, setOnceAttributes), - shouldCreatePerson, - }, - }; - }); - -export const toProcessorPersonEvent = ( - event: PersonSnapshotEventV1, -): ProcessorPersonEventV1Type => { - const optional: { - email?: string; - mergedIntoPersonId?: string; - name?: string; - primaryDistinctId?: string; - } = {}; - if (event.email) optional.email = event.email; - if (event.mergedIntoPersonId) optional.mergedIntoPersonId = event.mergedIntoPersonId; - if (event.name) optional.name = event.name; - if (event.primaryDistinctId) optional.primaryDistinctId = event.primaryDistinctId; - return { - changedAt: event.changedAt, - personId: event.personId, - ...optional, - isArchived: event.isArchived, - projectId: event.projectId, - schemaVersion: event.schemaVersion, - traits: event.traits, - version: event.version, - }; -}; - -const toProcessorPersonIdentityEvent = ({ - identityDistinctId, - mappingEvent, -}: { - readonly identityDistinctId: string; - readonly mappingEvent: PersonIdentityEventV1; -}): ProcessorPersonIdentityEventV1Type => { - // Prefer the explicit override direction from the synchronous merge; fall back - // to inferring it from the identify target for legacy mapping events. - const inferPreviousDistinctId = (): string | undefined => { - if (mappingEvent.distinctId === identityDistinctId) return undefined; - return mappingEvent.distinctId; - }; - const previousDistinctId = mappingEvent.previousDistinctId ?? inferPreviousDistinctId(); - const optional: { previousDistinctId?: string } = {}; - if (previousDistinctId) optional.previousDistinctId = previousDistinctId; - return { - changedAt: mappingEvent.changedAt, - personId: mappingEvent.personId, - distinctId: pick(Boolean(previousDistinctId), identityDistinctId, mappingEvent.distinctId), - isDeleted: mappingEvent.isDeleted, - ...optional, - projectId: mappingEvent.projectId, - schemaVersion: mappingEvent.schemaVersion, - version: mappingEvent.version, - }; -}; - -export const toProcessorPersonIdentityEvents = ( - identityResult: Pick, -): ReadonlyArray => - identityResult.mappingEvents.map((mappingEvent) => - toProcessorPersonIdentityEvent({ - identityDistinctId: identityResult.identity.distinctId, - mappingEvent, - }), - ); - -export class EventProcessorService extends Context.Service()( - "EventProcessorService", - { - make: Effect.gen(function* () { - const db = yield* Db; - const dlqProducer = yield* DlqProducer; - const personIdentityService = yield* PersonIdentityService; - - const emptyOutputs = (): ProcessorOutputs => ({ - personEvents: [], - personIdentityEvents: [], - processedEvents: [], - }); - - const processRecordToOutputs = Effect.fn("processRecordToOutputs")( - function* (transportRecord: CapturedTransportRecord) { - const now = yield* DateTime.nowAsDate; - - const capturedEvent = transportRecord.capturedEvent; - if (capturedEvent.captureId) - yield* Effect.annotateCurrentSpan("voidhash.capture.id", capturedEvent.captureId); - if (capturedEvent.projectId) - yield* Effect.annotateCurrentSpan("voidhash.project.id", capturedEvent.projectId); - if (capturedEvent.distinctId) - yield* Effect.annotateCurrentSpan( - "voidhash.person.distinct_id", - capturedEvent.distinctId, - ); - if (capturedEvent.event) - yield* Effect.annotateCurrentSpan("voidhash.event.name", capturedEvent.event); - if (capturedEvent.sessionId) - yield* Effect.annotateCurrentSpan("voidhash.session.id", capturedEvent.sessionId); - yield* Effect.annotateCurrentSpan("voidhash.processor.lane", transportRecord.lane); - - // Trusted revenue is server-stamped, so resolve it by `projectId` - // directly — its token may be synthetic (no `api_key` row). SDK events - // MUST resolve through the `api_key` join (token-spoofing protection). - const isTrustedRevenue = - capturedEvent.identityClaim?._tag === "Resolved" && - capturedEvent.routing.targetTopic === REVENUE_TRUSTED_SOURCE_TOPIC; - - const resolved = yield* Effect.result( - Effect.gen(function* () { - const projectRecord = yield* Effect.gen(function* () { - if (isTrustedRevenue) { - const [trustedRecord] = yield* db - .select({ - organizationId: projects.organizationId, - projectId: projects.id, - }) - .from(projects) - .where(eq(projects.id, capturedEvent.projectId)) - .limit(1); - return trustedRecord; - } - const [apiKeyRecord] = yield* db - .select({ - organizationId: projects.organizationId, - projectId: apiKeys.projectId, - }) - .from(apiKeys) - .innerJoin(projects, eq(projects.id, apiKeys.projectId)) - .where(and(eq(apiKeys.isPublic, true), eq(apiKeys.key, capturedEvent.token))) - .limit(1); - return apiKeyRecord; - }); - - if (!projectRecord) return null; - - const [policyRecord] = yield* db - .select() - .from(captureProjectPolicies) - .where(eq(captureProjectPolicies.projectId, projectRecord.projectId)) - .limit(1); - - let policy: ProcessorProjectPolicy = DEFAULT_PROCESSOR_POLICY; - if (policyRecord) { - policy = { - processorAllowHistorical: policyRecord.processorAllowHistorical, - processorAllowOverflow: policyRecord.processorAllowOverflow, - processorEnabled: policyRecord.processorEnabled, - processorHistoricalMinAgeHours: policyRecord.processorHistoricalMinAgeHours, - processorPersonProcessingEnabled: policyRecord.processorPersonProcessingEnabled, - processorSchemaMode: policyRecord.processorSchemaMode, - }; - } - - return { - organizationId: projectRecord.organizationId, - policy, - projectId: projectRecord.projectId, - }; - }), - ); - - if (resolved._tag === "Failure" || resolved.success === null) { - yield* dlqProducer.publishBatch([ - buildDlqEvent({ - captureId: transportRecord.capturedEvent.captureId, - distinctId: transportRecord.capturedEvent.distinctId, - failureClass: "project_not_found", - failureMessage: "failed to resolve processor project policy", - headers: transportRecord.headers, - lane: transportRecord.lane, - projectId: transportRecord.capturedEvent.projectId, - rawValue: transportRecord.rawValue, - sourceOffset: transportRecord.sourceOffset, - sourcePartition: transportRecord.sourcePartition, - sourceTopic: transportRecord.sourceTopic, - token: transportRecord.capturedEvent.token, - }), - ]); - return emptyOutputs(); - } - - if (resolved.success.organizationId) - yield* Effect.annotateCurrentSpan( - "voidhash.organization.id", - resolved.success.organizationId, - ); - - const attached = attachProjectPolicy({ - now, - record: transportRecord, - resolvedProject: resolved.success, - }); - if (!attached.ok) { - yield* dlqProducer.publishBatch([attached.value]); - return emptyOutputs(); - } - - if ( - isReservedRevenueEventName(attached.value.capturedEvent.event) && - attached.value.sourceTopic !== REVENUE_TRUSTED_SOURCE_TOPIC - ) { - yield* dlqProducer.publishBatch([ - buildDlqEvent({ - captureId: attached.value.capturedEvent.captureId, - distinctId: attached.value.capturedEvent.distinctId, - failureClass: "reserved_event_name", - failureMessage: `reserved revenue event '${attached.value.capturedEvent.event}' from untrusted source topic '${attached.value.sourceTopic}'`, - headers: attached.value.headers, - lane: attached.value.lane, - projectId: attached.value.capturedEvent.projectId, - rawKey: attached.value.rawKey, - rawValue: attached.value.rawValue, - sourceOffset: attached.value.sourceOffset, - sourcePartition: attached.value.sourcePartition, - sourceTopic: attached.value.sourceTopic, - token: attached.value.capturedEvent.token, - }), - ]); - return emptyOutputs(); - } - - // Honour a pre-resolved identity claim on trusted revenue: skip - // identity resolution and emit NO person/identity rows. Gated on the - // trusted source topic so a forged claim on an untrusted event falls - // through to normal resolution (defence-in-depth). - const identityClaim = attached.value.capturedEvent.identityClaim; - if ( - identityClaim?._tag === "Resolved" && - attached.value.sourceTopic === REVENUE_TRUSTED_SOURCE_TOPIC - ) { - const processedEvent = buildProcessedEvent({ - capturedEvent: attached.value.capturedEvent, - identity: { - distinctId: identityClaim.distinctId, - mode: "full", - personId: identityClaim.personId, - }, - lane: attached.value.lane, - sourceOffset: attached.value.sourceOffset, - sourcePartition: attached.value.sourcePartition, - sourceTopic: attached.value.sourceTopic, - }); - return { - personEvents: [], - personIdentityEvents: [], - processedEvents: [processedEvent], - } satisfies ProcessorOutputs; - } - - // Malformed person traits stay a defect here, exactly as the - // previous synchronous `throw` did. - const call = yield* Effect.orDie(buildPersonIdentityCall(attached.value)); - const identityResult = yield* Effect.gen(function* () { - if (call.kind === "identify") { - return yield* personIdentityService.identifyDistinctId(call.input); - } - return yield* personIdentityService.resolveDistinctId(call.input); - }); - - if (identityResult.identity.personId) - yield* Effect.annotateCurrentSpan( - "voidhash.person.id", - identityResult.identity.personId, - ); - - const processedEvent = buildProcessedEvent({ - capturedEvent: attached.value.capturedEvent, - identity: identityResult.identity, - lane: attached.value.lane, - sourceOffset: attached.value.sourceOffset, - sourcePartition: attached.value.sourcePartition, - sourceTopic: attached.value.sourceTopic, - }); - const personEvents = identityResult.personEvents.map(toProcessorPersonEvent); - const personIdentityEvents = toProcessorPersonIdentityEvents(identityResult); - - return { - personIdentityEvents, - personEvents, - processedEvents: [processedEvent], - } satisfies ProcessorOutputs; - }, - (effect) => - effect.pipe( - Effect.withSpan("event-processor.processRecordToOutputs"), - Effect.catchTags({ - DlqProducerError: (error) => - Effect.fail( - new EventProcessorServiceError({ - cause: String(error.cause ?? error.message), - message: error.message, - }), - ), - PersonServiceError: (error) => - Effect.fail( - new EventProcessorServiceError({ - cause: String(error.cause), - message: "identity resolution failed", - }), - ), - }), - ), - ); - - return constant({ processRecordToOutputs }); - }), - }, -) { - static readonly layer: Layer.Layer< - EventProcessorService, - never, - Db | DlqProducer | PersonIdentityService - > = Layer.effect(EventProcessorService)(EventProcessorService.make); -} diff --git a/packages/core/src/services/analyticsIngest/PolicyCounterStore.ts b/packages/core/src/services/analyticsIngest/PolicyCounterStore.ts deleted file mode 100644 index bb36643b8..000000000 --- a/packages/core/src/services/analyticsIngest/PolicyCounterStore.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Abstract port for the capture policy counters (requests/minute, events/day). - * - * The application root picks the implementation: a KV-backed live adapter - * (best-effort `get`+`put`, eventually consistent) or {@link PolicyCounterStore.noop} - * (always-allow). Projects that need strict limits should use the noop layer - * plus an upstream Cloudflare WAF rule. The concrete KV adapter lives at the - * app root so `packages/core` carries no infrastructure dependency. - * - * `PlatformRuntime` preserves the guarantee that runtime-backed counter - * implementations only run inside a configured runtime, without coupling this - * port to a provider implementation. - */ -import { Context, Effect, Layer, Schema } from "effect"; -import type { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; - -export class PolicyStoreError extends Schema.TaggedErrorClass("PolicyStoreError")( - "PolicyStoreError", - { - message: Schema.String, - cause: Schema.optional(Schema.String), - }, -) {} - -export interface RequestLimitCheck { - readonly allowed: boolean; - readonly retryAfterMs?: number; -} - -export interface PolicyCounterStoreShape { - readonly checkRequestLimit: (input: { - readonly now: Date; - readonly projectId: string; - readonly requestsPerMinute: number | undefined; - }) => Effect.Effect; - - readonly checkEventQuota: (input: { - readonly now: Date; - readonly projectId: string; - readonly quota: number | undefined; - }) => Effect.Effect; -} - -export class PolicyCounterStore extends Context.Service< - PolicyCounterStore, - PolicyCounterStoreShape ->()("@voidhash/core/PolicyCounterStore") { - /** Always-allow store — for tests, or apps that defer rate limiting to a WAF. */ - static readonly noop: Layer.Layer = Layer.succeed(PolicyCounterStore, { - checkRequestLimit: () => Effect.succeed({ allowed: true }), - checkEventQuota: () => Effect.succeed(true), - }); -} diff --git a/packages/core/src/services/analyticsIngest/ProcessorOutputs.ts b/packages/core/src/services/analyticsIngest/ProcessorOutputs.ts deleted file mode 100644 index 174af581b..000000000 --- a/packages/core/src/services/analyticsIngest/ProcessorOutputs.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { - ProcessedEventV2Type, - ProcessorPersonEventV1Type, - ProcessorPersonIdentityEventV1Type, -} from "../../domain/analyticsIngest/AnalyticsIngest.ts"; - -/** Analytics processor outputs collected by a durable ingest flush workflow. */ -export interface ProcessorOutputs { - readonly processedEvents: ReadonlyArray; - readonly personEvents: ReadonlyArray; - readonly personIdentityEvents: ReadonlyArray; -} diff --git a/packages/core/src/services/index.ts b/packages/core/src/services/index.ts index 365ba05b7..1bad01041 100644 --- a/packages/core/src/services/index.ts +++ b/packages/core/src/services/index.ts @@ -1,16 +1,7 @@ export * from "./analytics/AnalyticsService.ts"; -export * from "./analytics/CustomAnalyticsService.ts"; -export * from "./voidql/VoidQlService.ts"; +export * from "./analytics/AnalyticsEventStore.ts"; export * from "./analyticsIngest/AnalyticsDispatchService.ts"; -export * from "./analyticsIngest/AnalyticsIngestDlqService.ts"; -export * from "./analyticsIngest/AnalyticsJanitorService.ts"; -export * from "./analyticsIngest/AnalyticsWriterService.ts"; -export * from "./analyticsIngest/CaptureIngress.ts"; -export * from "./analyticsIngest/DlqProducer.ts"; export * from "./analyticsIngest/EventCaptureService.ts"; -export * from "./analyticsIngest/EventProcessorService.ts"; -export * from "./analyticsIngest/PolicyCounterStore.ts"; -export * from "./analyticsIngest/ProcessorOutputs.ts"; export * from "./apiKeys/ApiKeyService.ts"; export * from "./auditLog/AuditLogPort.ts"; export * from "./auth/AuthTokenVerifier.ts"; diff --git a/packages/core/src/services/organizations/OrganizationService.ts b/packages/core/src/services/organizations/OrganizationService.ts index 5c4b30a90..9d567cee5 100644 --- a/packages/core/src/services/organizations/OrganizationService.ts +++ b/packages/core/src/services/organizations/OrganizationService.ts @@ -35,11 +35,6 @@ export class OrganizationServiceError extends Schema.TaggedErrorClass; } -/** Only the optional snapshot fields that actually carry a value. */ -const personOptionalFields = ( - event: PersonSnapshotEventV1, -): Pick< - Partial, - "email" | "mergedIntoPersonId" | "name" | "primaryDistinctId" -> => { - const fields: { - email?: string; - mergedIntoPersonId?: string; - name?: string; - primaryDistinctId?: string; - } = {}; - if (event.email) fields.email = event.email; - if (event.mergedIntoPersonId) fields.mergedIntoPersonId = event.mergedIntoPersonId; - if (event.name) fields.name = event.name; - if (event.primaryDistinctId) fields.primaryDistinctId = event.primaryDistinctId; - return fields; -}; - -const toProcessorPersonEvent = (event: PersonSnapshotEventV1): ProcessorPersonEventV1 => ({ - changedAt: event.changedAt, - personId: event.personId, - isArchived: event.isArchived, - projectId: event.projectId, - schemaVersion: event.schemaVersion, - traits: event.traits, - version: event.version, - ...personOptionalFields(event), -}); - -const toProcessorPersonIdentityEvent = ({ - identityDistinctId, - mappingEvent, -}: { - readonly identityDistinctId: string; - readonly mappingEvent: PersonIdentityEventV1; -}): ProcessorPersonIdentityEventV1 => { - const base = { - changedAt: mappingEvent.changedAt, - personId: mappingEvent.personId, - isDeleted: mappingEvent.isDeleted, - projectId: mappingEvent.projectId, - schemaVersion: mappingEvent.schemaVersion, - version: mappingEvent.version, - }; - - // A mapping event on the identity's own distinct id is not an alias, so it - // carries no previous distinct id. - if (!mappingEvent.distinctId || mappingEvent.distinctId === identityDistinctId) { - return { ...base, distinctId: mappingEvent.distinctId }; - } - - return { - ...base, - distinctId: identityDistinctId, - previousDistinctId: mappingEvent.distinctId, - }; -}; - -/** - * Publishes identity projection events produced outside the capture flush - * batch. The analytics ingest path uses {@link noop} because its processor - * returns these events to its batch writer; the SDK composition uses - * {@link analyticsWriterLayer} for direct ClickHouse writes. - */ +/** Optional edition port for projecting identity mutations into analytics. */ export class IdentityProjectionPublisher extends Context.Service< IdentityProjectionPublisher, { @@ -120,62 +50,15 @@ export class IdentityProjectionPublisher extends Context.Service< ) => Effect.Effect; } >()("IdentityProjectionPublisher", { - make: Effect.sync(() => { - return { publishIdentityResult: () => Effect.void }; - }), + make: Effect.sync(() => ({ publishIdentityResult: () => Effect.void })), }) { static readonly layer = Layer.effect(IdentityProjectionPublisher)( IdentityProjectionPublisher.make, ); - static readonly analyticsWriterLayer = Layer.effect( - IdentityProjectionPublisher, - Effect.gen(function* () { - const writer = yield* AnalyticsWriterService; - - return { - publishIdentityResult: (input) => { - const personMessages = input.personEvents.map(toProcessorPersonEvent); - const identityMessages = input.mappingEvents.map((mappingEvent) => - toProcessorPersonIdentityEvent({ - identityDistinctId: input.identity.distinctId, - mappingEvent, - }), - ); - - return writer - .writeMessages([ - ...personMessages.map((person) => ({ - kind: constant("person"), - messageId: `${person.projectId}:${person.personId}:${person.version}`, - value: person, - })), - ...identityMessages.map((identity) => ({ - kind: constant("person-distinct-id"), - messageId: `${identity.projectId}:${identity.distinctId}:${identity.version}`, - value: identity, - })), - ]) - .pipe( - Effect.asVoid, - Effect.mapError( - (error) => - new QueueProducerError({ - cause: error.cause, - queueName: "AnalyticsWriterService", - }), - ), - ); - }, - }; - }), - ); - - /** No-op variant used by tests and capture flush paths that return outputs directly. */ + /** Community implementation; PostgreSQL analytics does not project identity tables. */ static readonly noop: Layer.Layer = Layer.succeed( IdentityProjectionPublisher, - { - publishIdentityResult: () => Effect.void, - }, + { publishIdentityResult: () => Effect.void }, ); } diff --git a/packages/core/src/services/purchaseProcessing/PurchaseLedgerWorkerService.ts b/packages/core/src/services/purchaseProcessing/PurchaseLedgerWorkerService.ts index 0674e3fdc..4e9b716fb 100644 --- a/packages/core/src/services/purchaseProcessing/PurchaseLedgerWorkerService.ts +++ b/packages/core/src/services/purchaseProcessing/PurchaseLedgerWorkerService.ts @@ -1,9 +1,9 @@ /** * `PurchaseLedgerWorkerService` drains the `purchase_ledger` table — written * transactionally by `PurchaseProcessingService` — by re-dispatching each row's - * `eventsPayload` onto the SHARED analytics-ingest queue via + * `eventsPayload` through * `AnalyticsDispatchService.dispatchTrusted`. The ledger is the durability - * backstop: it guarantees every revenue event is eventually enqueued even if an + * backstop: it guarantees every revenue event is eventually dispatched even if an * immediate post-commit dispatch is lost. * * Driver: on the Cloudflare backend a cron-triggered `PurchaseLedgerDrainWorkflow` @@ -14,12 +14,9 @@ * given row at a time. No leader election needed. Stale claims (worker crashed * mid-row) are swept back to `Pending` at the top of each poll. * - * Worker dispatch ≠ analytics delivery. The worker's job ends at "I enqueued - * the batch onto the at-least-once analytics queue"; the processor + writer - * downstream perform identity pass-through and the ClickHouse insert, and the - * writer's unbounded `(project_id, event_id)` pre-check collapses the duplicates - * that the immediate-dispatch + drain overlap (and queue retries) produce — - * `eventId` is deterministic, so a re-dispatch is a no-op, never a double-write. + * `AnalyticsDispatchService` is edition-specific: Community persists the batch + * synchronously, while hosted runtimes may enqueue it. Deterministic event ids + * make the immediate-dispatch and ledger-drain overlap idempotent in both cases. */ import { Cause, Context, Effect, Layer, Schedule, Schema } from "effect"; diff --git a/packages/core/src/services/purchaseProcessing/revenue-analytics-mapper.ts b/packages/core/src/services/purchaseProcessing/revenue-analytics-mapper.ts index 6cbd7972c..eab8028c0 100644 --- a/packages/core/src/services/purchaseProcessing/revenue-analytics-mapper.ts +++ b/packages/core/src/services/purchaseProcessing/revenue-analytics-mapper.ts @@ -4,7 +4,7 @@ * `eventId` is a DETERMINISTIC id derived from `(idempotencyKey, eventName, * personId)` via {@link deterministicAnalyticsEventId} — stable across retries * and re-dispatch, so the at-least-once analytics queue dedupes duplicates on - * the ClickHouse `(project_id, event_id)` key rather than relying on a single + * the portable `(project_id, event_id)` key rather than relying on a single * upstream invocation. The ledger `idempotency_key` is the per-action anchor * (one ledger row ⇒ one mapper call), and `(eventName, personId)` disambiguates * the multi-event actions (a subscription start emits two named events; a diff --git a/packages/core/src/services/sdk/SdkService.ts b/packages/core/src/services/sdk/SdkService.ts index 427af4402..71c4985f0 100644 --- a/packages/core/src/services/sdk/SdkService.ts +++ b/packages/core/src/services/sdk/SdkService.ts @@ -648,11 +648,10 @@ export class SdkService extends Context.Service()("SdkService", { yield* Effect.annotateCurrentSpan("voidhash.person.id", identityResult.identity.personId); - // Project the synchronous write into ClickHouse so analytics stays - // consistent with the operational Postgres row. Uses the projection - // publisher (person/identity rows only — no synthetic analytics event, - // no quota impact). A ClickHouse failure must not fail the durable - // Postgres write, so log-and-swallow (mirrors `identifyDistinctId`). + // Publish the identity projection after the durable person write. The + // Community publisher is a no-op; hosted runtimes may maintain an + // analytics-side identity model. Projection failures never roll back + // the operational PostgreSQL write. yield* identityProjectionPublisher .publishIdentityResult({ identity: { distinctId }, @@ -662,7 +661,7 @@ export class SdkService extends Context.Service()("SdkService", { .pipe( Effect.catch((error) => Effect.logError( - "Failed to project synchronous person-attribute write to analytics; Postgres is updated but ClickHouse will lag until a later event re-emits this person", + "Failed to project synchronous person-attribute write to analytics; PostgreSQL is updated and a later event may retry the projection", { cause: error, distinctId, personId: identityResult.identity.personId, projectId }, ), ), diff --git a/packages/core/src/services/voidql/VoidQlService.ts b/packages/core/src/services/voidql/VoidQlService.ts deleted file mode 100644 index 8475d340c..000000000 --- a/packages/core/src/services/voidql/VoidQlService.ts +++ /dev/null @@ -1,413 +0,0 @@ -/** - * {@link VoidQlService} — the public VoidQL surface (§13): `runQuery`, - * `validateQuery`, `getSchema`, `saveInsight`. - * - * The hot path is `authorize → buildAuthorizedScope → compile (pure ①–⑥, incl. - * verify) → execute → audit`. Scope is derived **server-side** from the - * authenticated session (never the request body) and inlined as the bound - * `{pOrg}`/`{pPids}` literals by the compiler — there is no tenant setting for a - * jailbroken agent to re-point (§14). Execution runs under the locked-down - * `analytics_query` ClickHouse user with a server-random `withQueryId` (the KILL - * handle) and a stable per-principal `withQuotaKey` (so the shared user's - * `KEYED BY client_key` quota isolates tenants); `readonly=1` + the CONST caps come - * from the profile, NOT a per-request setting (§7 L3). - * - * ClickHouse execution errors are mapped to a constant {@link VoidQlExecutionError} - * — never forwarding raw messages or row counts, which would be a side-channel - * (§18 gap #7). When ClickHouse is not configured, authorized queries fail - * closed to an empty result while schema, validation, and saved-query metadata - * remain available. - */ -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { Db, analyticsDashboardItems, analyticsSavedQuery, and, desc, eq } from "@voidhash/db"; -import { constant } from "@voidhash/lib/lang"; -import { Context, Effect, Layer, Option } from "effect"; - -import { AuthSession } from "../../domain/auth/Auth.ts"; -import { checkOrganizationPermission } from "../../utils/permissions.ts"; -import { generateId } from "../../utils/generate-id.ts"; -import { CATALOG, CATALOG_SCHEMA_VERSION } from "./catalog/index.ts"; -import type { Capability, ColumnSpec } from "./catalog/types.ts"; -import { compileVoidQl } from "./compile.ts"; -import { - type Diagnostic, - isVoidQlCompileError, - toDiagnostic, - VoidQlExecutionError, -} from "./errors.ts"; -import { registeredFunctionNames } from "./functions.ts"; -import { toStatement } from "./ir.ts"; -import { type AuthorizedScope, makeAuthorizedScope } from "./scope.ts"; - -export interface VoidQlPrincipal { - readonly kind: "user" | "agent"; - readonly id: string; -} - -export interface RunQueryInput { - readonly organizationId: string; - readonly text: string; - readonly principal: VoidQlPrincipal; -} - -export interface RunQueryResult { - readonly columns: readonly ColumnSpec[]; - readonly rows: ReadonlyArray>; -} - -export interface ValidateResult { - readonly valid: boolean; - readonly columns?: readonly ColumnSpec[]; - readonly diagnostic?: Diagnostic; -} - -export interface SchemaDescriptor { - readonly dialect: string; - readonly tables: ReadonlyArray<{ - readonly name: string; - readonly columns: ReadonlyArray<{ - readonly name: string; - readonly type: string; - readonly pii: boolean; - readonly doc: string; - }>; - readonly namespaces: ReadonlyArray<{ - readonly name: string; - readonly pii: boolean; - readonly doc: string; - }>; - }>; - readonly functions: readonly string[]; -} - -type AnalyticsSavedQueryRow = typeof analyticsSavedQuery.$inferSelect; - -const toSavedInsight = (row: AnalyticsSavedQueryRow) => ({ - createdAt: row.createdAt, - createdBy: row.createdBy, - id: row.id, - name: row.name, - organizationId: row.organizationId, - schemaVersion: row.schemaVersion, - text: row.voidqlText, - updatedAt: row.updatedAt, -}); - -const DIALECT_REFERENCE = - "VoidQL is a read-only SQL subset over events|persons|revenue. " + - "SELECT … FROM … [JOIN … ON …] [WHERE] [GROUP BY] [HAVING] [ORDER BY] [LIMIT]; CTEs and subqueries supported. " + - "No organization_id, no SETTINGS, no table functions — tenant scope is applied automatically."; - -/** Cheap, non-cryptographic FNV-1a hash for the audit log (text fingerprint only). */ -const fnv1a = (text: string): string => { - let hash = 0x811c9dc5; - for (let i = 0; i < text.length; i++) { - hash ^= text.charCodeAt(i); - hash = Math.imul(hash, 0x01000193); - } - return (hash >>> 0).toString(16); -}; - -const buildSchemaDescriptor = (): SchemaDescriptor => ({ - dialect: DIALECT_REFERENCE, - tables: Object.values(CATALOG).map((table) => ({ - name: table.name, - columns: Object.values(table.columns).map((c) => ({ - name: c.name, - type: c.type, - pii: c.requires.includes("pii"), - doc: c.doc, - })), - namespaces: Object.values(table.namespaces).map((n) => ({ - name: n.name, - pii: n.requires.includes("pii"), - doc: n.doc, - })), - })), - functions: registeredFunctionNames(), -}); - -export class VoidQlService extends Context.Service()("VoidQlService", { - make: Effect.gen(function* () { - // The locked-down `analytics_query` user (readonly=1 + CONST caps + no row - // policy). In production the VoidQL RPC path must be provided this client as - // its ambient ClickhouseWebClient — running under the RLS readonly user would - // fail-closed to zero rows, since VoidQL injects no `SQL_organization_id`. - const ch = Option.getOrUndefined( - yield* Effect.serviceOption(ClickhouseWebClient.ClickhouseWebClient), - ); - const db = yield* Db; - - /** Authorize the claimed org against the session, then derive scope from Postgres. */ - const buildAuthorizedScope = Effect.fn("voidql.buildScope")(function* (organizationId: string) { - yield* checkOrganizationPermission(organizationId, "organization:all", "VoidQL read denied"); - const projects = yield* db.query.projects.findMany({ - columns: { id: true }, - where: { organizationId }, - }); - return makeAuthorizedScope({ - organizationId, - availableProjectIds: projects.map((p) => p.id), - }); - }); - - // AI agents never get `pii` by default (§9, §14); authorized users do. - const capabilitiesFor = (principal: VoidQlPrincipal): ReadonlySet => { - if (principal.kind === "user") return new Set(["pii"]); - return new Set([]); - }; - - const loadSavedInsight = Effect.fn("voidql.loadSavedInsight")(function* (id: string) { - const [insight] = yield* db - .select() - .from(analyticsSavedQuery) - .where(eq(analyticsSavedQuery.id, id)) - .limit(1); - if (!insight) { - return yield* Effect.fail( - new VoidQlExecutionError({ - cause: "not_found", - message: "The saved query was not found.", - }), - ); - } - yield* buildAuthorizedScope(insight.organizationId); - return insight; - }); - - const auditRun = ( - input: RunQueryInput, - scope: AuthorizedScope, - queryId: string, - rowCount: number, - ) => - Effect.gen(function* () { - yield* Effect.annotateCurrentSpan("voidhash.organization.id", input.organizationId); - yield* Effect.annotateCurrentSpan("voidhash.voidql.principal.kind", input.principal.kind); - yield* Effect.annotateCurrentSpan("voidhash.voidql.principal.id", input.principal.id); - yield* Effect.annotateCurrentSpan("voidhash.voidql.text_hash", fnv1a(input.text)); - yield* Effect.annotateCurrentSpan("voidhash.voidql.query_id", queryId); - yield* Effect.annotateCurrentSpan("voidhash.voidql.row_count", rowCount); - yield* Effect.annotateCurrentSpan( - "voidhash.voidql.project_ids.count", - scope.availableProjectIds.length, - ); - }); - - const runQuery = Effect.fn("voidql.runQuery")( - function* (input: RunQueryInput) { - const scope = yield* buildAuthorizedScope(input.organizationId); - // Fail-closed: an org with no readable projects can read nothing. - if (scope.availableProjectIds.length === 0) { - return { columns: [], rows: [] } satisfies RunQueryResult; - } - if (ch === undefined) { - return { columns: [], rows: [] } satisfies RunQueryResult; - } - const compiled = yield* compileVoidQl(input.text, scope, capabilitiesFor(input.principal)); - // Stable per-principal quota key: the shared `analytics_query` user's - // `KEYED BY client_key` DoS quota only isolates tenants when each request - // carries one (otherwise every request shares the empty-key global bucket). - const quotaKey = `${input.organizationId}:${input.principal.kind}:${input.principal.id}`; - const rows = yield* toStatement(ch, compiled.pieces).pipe( - ch.withQueryId(compiled.queryId), - ch.withQuotaKey(quotaKey), - ); - yield* auditRun(input, scope, compiled.queryId, rows.length); - return { columns: compiled.columns, rows } satisfies RunQueryResult; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - SqlError: () => - // Constant message — never forward CH internals / row counts (§18 #7). - Effect.fail( - new VoidQlExecutionError({ - cause: "clickhouse", - message: "The query could not be executed.", - }), - ), - EffectDrizzleQueryError: () => - Effect.fail( - new VoidQlExecutionError({ - cause: "database", - message: "The query could not be executed.", - }), - ), - }), - ), - ); - - const validateQuery = Effect.fn("voidql.validateQuery")( - function* (input: RunQueryInput) { - const scope = yield* buildAuthorizedScope(input.organizationId); - return yield* compileVoidQl(input.text, scope, capabilitiesFor(input.principal)).pipe( - Effect.map((compiled): ValidateResult => ({ valid: true, columns: compiled.columns })), - // User-facing compile errors are surfaced as diagnostics (data), not - // failures — this is what the agent repair loop / editor lint consume. - // BUT an isolation-verifier failure is a COMPILER DEFECT, never user - // error: it must alarm server-side (matching the run path, which maps it - // to an opaque execution error), not be folded into a benign {valid:false} - // diagnostic an attacker/repair-loop can iterate against — so we Effect.die. - Effect.catchIf( - isVoidQlCompileError, - (error): Effect.Effect => { - if (error._tag === "VoidQlIsolationError") return Effect.die(error); - return Effect.succeed({ valid: false, diagnostic: toDiagnostic(error) }); - }, - ), - ); - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: () => - Effect.fail( - new VoidQlExecutionError({ - cause: "database", - message: "The query could not be validated.", - }), - ), - }), - ), - ); - - const getSchema = Effect.fn("voidql.getSchema")(() => Effect.succeed(buildSchemaDescriptor())); - - const saveInsight = Effect.fn("voidql.saveInsight")( - function* (input: { - readonly organizationId: string; - readonly name: string; - readonly text: string; - }) { - const session = yield* AuthSession; - const scope = yield* buildAuthorizedScope(input.organizationId); - // Re-validate before persisting; a save of a non-compiling query is rejected. - yield* compileVoidQl(input.text, scope, new Set(["pii"])); - const id = generateId("analyticsSavedQuery"); - yield* db.insert(analyticsSavedQuery).values({ - id, - organizationId: input.organizationId, - name: input.name, - voidqlText: input.text, - schemaVersion: CATALOG_SCHEMA_VERSION, - createdBy: session?.user?.id ?? "system", - }); - return { id }; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: () => - Effect.fail( - new VoidQlExecutionError({ - cause: "database", - message: "The insight could not be saved.", - }), - ), - }), - ), - ); - - /** List saved VoidQL insights visible in an organization. */ - const listInsights = Effect.fn("voidql.listInsights")( - function* (input: { readonly organizationId: string }) { - yield* buildAuthorizedScope(input.organizationId); - const rows = yield* db - .select() - .from(analyticsSavedQuery) - .where(eq(analyticsSavedQuery.organizationId, input.organizationId)) - .orderBy(desc(analyticsSavedQuery.updatedAt)); - return { insights: rows.map(toSavedInsight) }; - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", () => - Effect.fail( - new VoidQlExecutionError({ - cause: "database", - message: "The saved queries could not be listed.", - }), - ), - ), - ), - ); - - /** Recompile and execute a saved VoidQL insight under the current authorization scope. */ - const runSavedInsight = Effect.fn("voidql.runSavedInsight")( - function* (input: { readonly id: string; readonly principal: VoidQlPrincipal }) { - const insight = yield* loadSavedInsight(input.id); - return yield* runQuery({ - organizationId: insight.organizationId, - principal: input.principal, - text: insight.voidqlText, - }); - }, - (effect) => - effect.pipe( - Effect.catchTag("EffectDrizzleQueryError", () => - Effect.fail( - new VoidQlExecutionError({ - cause: "database", - message: "The saved query could not be executed.", - }), - ), - ), - ), - ); - - /** Delete an authorized saved VoidQL insight. */ - const deleteInsight = Effect.fn("voidql.deleteInsight")( - function* (input: { readonly id: string }) { - const insight = yield* loadSavedInsight(input.id); - yield* db.transaction((tx) => - Effect.gen(function* () { - yield* tx.delete(analyticsSavedQuery).where(eq(analyticsSavedQuery.id, insight.id)); - yield* tx - .delete(analyticsDashboardItems) - .where( - and( - eq(analyticsDashboardItems.sourceType, "voidql"), - eq(analyticsDashboardItems.sourceId, insight.id), - ), - ); - }), - ); - return { deleted: true }; - }, - (effect) => - effect.pipe( - Effect.catchTags({ - EffectDrizzleQueryError: () => - Effect.fail( - new VoidQlExecutionError({ - cause: "database", - message: "The saved query could not be deleted.", - }), - ), - SqlError: () => - Effect.fail( - new VoidQlExecutionError({ - cause: "database", - message: "The saved query could not be deleted.", - }), - ), - }), - ), - ); - - return constant({ - deleteInsight, - getSchema, - listInsights, - runQuery, - runSavedInsight, - saveInsight, - validateQuery, - }); - }), -}) { - static layer: Layer.Layer = Layer.effect(VoidQlService)( - VoidQlService.make, - ); -} diff --git a/packages/core/src/services/voidql/ast/VoidQlAst.ts b/packages/core/src/services/voidql/ast/VoidQlAst.ts deleted file mode 100644 index 9fe48d065..000000000 --- a/packages/core/src/services/voidql/ast/VoidQlAst.ts +++ /dev/null @@ -1,292 +0,0 @@ -/** - * The VoidQL abstract syntax tree. - * - * Plain-TypeScript `readonly` tagged-union interfaces discriminated by `_tag` — - * deliberately **no `Schema` and no decode step** (see `docs/analytics-access-layer.html` - * §8.2). The parser ({@link file://../parser.ts}) is the *sole, trusted constructor* - * of these nodes; the tree is only ever built server-side from query text and is - * never deserialised from foreign JSON, so a `Schema.decode` boundary would guard - * an input that cannot occur. The one guarantee that matters — exhaustiveness — is - * obtained at *compile time* from the printer's `default: node satisfies never`, - * which is strictly stronger than TRQL's runtime `NotImplementedError` and HogQL's - * runtime `getattr` dispatch. - * - * There is intentionally **no raw-SQL node**: user text can never travel to - * `ch.literal`. Field-level validity (e.g. `numType` ∈ three values, a non-empty - * column `chain`) is the parser's and resolver's responsibility, since no runtime - * boundary re-checks it. - */ - -/** A source position, used to render caret-precise diagnostics. */ -export interface Pos { - readonly line: number; - readonly col: number; - readonly offset: number; -} - -/** A source span (half-open: `[start, end)`), carried by every node. */ -export interface Span { - readonly start: Pos; - readonly end: Pos; -} - -interface Node { - readonly span: Span; -} - -// ─────────────────────────────── expressions ──────────────────────────────── - -export interface StringLit extends Node { - readonly _tag: "StringLit"; - readonly value: string; -} - -/** - * A numeric literal. `numType` carries the resolved scalar type so the printer - * can emit an *explicit* `ch.param` type — the substrate would otherwise infer a - * JS `number` as `Decimal`, degrading partition pruning (§18 gap #9). Validity of - * `numType` is the parser's responsibility; no decode boundary re-checks it. - */ -export interface NumberLit extends Node { - readonly _tag: "NumberLit"; - readonly value: number; - readonly numType: "Int64" | "UInt64" | "Float64"; -} - -export interface BoolLit extends Node { - readonly _tag: "BoolLit"; - readonly value: boolean; -} - -export interface NullLit extends Node { - readonly _tag: "NullLit"; -} - -/** - * A column reference as a dotted chain (HogQL's `Field(chain=[...])`). Whether - * `a.b` is `table.column` or a JSON-property access is decided by the *resolver*, - * never the parser. - */ -export interface ColumnRef extends Node { - readonly _tag: "ColumnRef"; - readonly chain: readonly [string, ...string[]]; -} - -export interface StarRef extends Node { - readonly _tag: "StarRef"; - readonly qualifier?: string; -} - -/** - * A function call. The `name` is just a token here; the resolver checks it - * against the closed function registry (§11) so the allow-list is single-sourced. - */ -export interface FnCall extends Node { - readonly _tag: "FnCall"; - readonly name: string; - readonly args: readonly Expr[]; -} - -export type WindowFrameBound = "unboundedPreceding" | "currentRow" | "unboundedFollowing"; - -export interface WindowFrame { - readonly unit: "rows" | "range"; - readonly start: WindowFrameBound; - readonly end?: WindowFrameBound; -} - -export interface WindowExpr extends Node { - readonly _tag: "WindowExpr"; - readonly fn: FnCall; - readonly partitionBy: readonly Expr[]; - readonly orderBy: readonly OrderItem[]; - readonly frame?: WindowFrame; -} - -export type BinaryOp = - | "or" - | "and" - | "eq" - | "neq" - | "lt" - | "lte" - | "gt" - | "gte" - | "like" - | "notLike" - | "ilike" - | "notIlike" - | "add" - | "sub" - | "mul" - | "div" - | "mod"; - -export interface Binary extends Node { - readonly _tag: "Binary"; - readonly op: BinaryOp; - readonly left: Expr; - readonly right: Expr; -} - -export interface Unary extends Node { - readonly _tag: "Unary"; - readonly op: "not" | "neg"; - readonly expr: Expr; -} - -export interface InExpr extends Node { - readonly _tag: "InExpr"; - readonly expr: Expr; - readonly list?: readonly [Expr, ...Expr[]]; - readonly query?: Query; - readonly negated: boolean; -} - -export interface ExistsExpr extends Node { - readonly _tag: "ExistsExpr"; - readonly query: Query; -} - -export interface SubqueryExpr extends Node { - readonly _tag: "SubqueryExpr"; - readonly query: Query; -} - -export interface Between extends Node { - readonly _tag: "Between"; - readonly expr: Expr; - readonly low: Expr; - readonly high: Expr; - readonly negated: boolean; -} - -export interface IsNull extends Node { - readonly _tag: "IsNull"; - readonly expr: Expr; - readonly negated: boolean; -} - -export interface CaseWhen { - readonly when: Expr; - readonly then: Expr; -} - -export interface CaseExpr extends Node { - readonly _tag: "CaseExpr"; - readonly operand?: Expr; - readonly whens: readonly [CaseWhen, ...CaseWhen[]]; - readonly else?: Expr; -} - -export interface Paren extends Node { - readonly _tag: "Paren"; - readonly expr: Expr; -} - -export type Expr = - | StringLit - | NumberLit - | BoolLit - | NullLit - | ColumnRef - | StarRef - | FnCall - | WindowExpr - | Binary - | Unary - | InExpr - | ExistsExpr - | SubqueryExpr - | Between - | IsNull - | CaseExpr - | Paren; - -// ─────────────────────────────── statements ───────────────────────────────── - -export interface SelectItem extends Node { - readonly _tag: "SelectItem"; - readonly expr: Expr; - readonly alias?: string; -} - -/** - * A named `FROM`/`JOIN` source. There is deliberately no `db.table` and no table - * function — those have *no grammar production at all*, so the CVE-2025-1520 - * table-function class is structurally unreachable (§8.1, §11). - */ -export interface NamedTable extends Node { - readonly _tag: "NamedTable"; - readonly name: string; - readonly alias?: string; -} - -/** A subquery source. Subqueries MUST be aliased (the parser enforces it). */ -export interface SubquerySource extends Node { - readonly _tag: "SubquerySource"; - readonly query: Query; - readonly alias: string; -} - -export type TableSource = NamedTable | SubquerySource; - -export interface Join extends Node { - readonly _tag: "Join"; - readonly kind: "inner" | "left" | "right" | "full" | "cross"; - readonly source: TableSource; - readonly on?: Expr; - readonly using?: readonly [string, ...string[]]; -} - -export interface OrderItem extends Node { - readonly _tag: "OrderItem"; - readonly expr: Expr; - readonly dir: "asc" | "desc"; - readonly nulls?: "first" | "last"; -} - -export interface Cte extends Node { - readonly _tag: "Cte"; - readonly name: string; - readonly query: Query; -} - -export interface LimitBy { - readonly limit: number; - readonly offset?: number; - readonly by: readonly [Expr, ...Expr[]]; -} - -export interface Select extends Node { - readonly _tag: "Select"; - readonly with: readonly Cte[]; - readonly distinct: boolean; - readonly distinctOn: readonly Expr[]; - readonly columns: readonly [SelectItem, ...SelectItem[]]; - readonly from?: TableSource; - readonly joins: readonly Join[]; - readonly prewhere?: Expr; - readonly where?: Expr; - readonly groupBy: readonly Expr[]; - readonly groupByModifier?: "rollup" | "cube"; - readonly withTotals: boolean; - readonly having?: Expr; - readonly qualify?: Expr; - readonly orderBy: readonly OrderItem[]; - readonly limitBy?: LimitBy; - readonly limit?: number; - readonly offset?: number; - readonly withTies: boolean; -} - -export type SetOperator = "UNION ALL" | "UNION DISTINCT" | "INTERSECT" | "EXCEPT"; - -export interface SetQuery extends Node { - readonly _tag: "SetQuery"; - readonly selects: readonly [Select, Select, ...Select[]]; - readonly operators: readonly [SetOperator, ...SetOperator[]]; -} - -export type Query = Select | SetQuery; -export type Statement = Query; diff --git a/packages/core/src/services/voidql/catalog/brand.ts b/packages/core/src/services/voidql/catalog/brand.ts deleted file mode 100644 index 21b2b1b33..000000000 --- a/packages/core/src/services/voidql/catalog/brand.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * The type-level barrier that makes "no user string ever reaches `ch.literal`" - * a compiler-checked invariant (§12). {@link CatalogSql} is a branded string; - * the printer's `lit()` accepts only the brand, and {@link catalog} — the sole - * constructor — is fed exclusively from frozen catalog/keyword constants and - * already-validated identifiers, never from user text. A lint/CODEOWNERS rule - * keeps `ch.literal(` and `catalog(` confined to the VoidQL printer + catalog. - */ -import { Brand, Schema } from "effect"; - -export const CatalogSqlSchema = Schema.String.pipe(Schema.brand("CatalogSql")); - -export type CatalogSql = typeof CatalogSqlSchema.Type; - -/** Brand a compiler-controlled SQL fragment. NEVER call with user-derived text. */ -export const catalog = Brand.nominal(); diff --git a/packages/core/src/services/voidql/catalog/events.ts b/packages/core/src/services/voidql/catalog/events.ts deleted file mode 100644 index f3de1f6e9..000000000 --- a/packages/core/src/services/voidql/catalog/events.ts +++ /dev/null @@ -1,144 +0,0 @@ -/** - * The `events` logical view (§9). Lowers to the audited - * `resolvedEventsFrom` + `RESOLVED_EVENTS_JOIN` shape — auto-deduped - * (`LIMIT 1 BY event_id ORDER BY processed_ts DESC`) and identity-resolved - * (`effective*IdExpression`) — so the raw `events_v2` MergeTree is never - * reachable. The dedup/identity SQL is imported verbatim from - * `clickhouse-accessor.ts` (single source of truth); VoidQL differs in *one* - * way: the tenant predicate is **inlined as bound literals into the inner WHERE** - * (not applied via the `tenantSettings` row-policy setting — the P1 blocker, §20). - */ -import { - CLICKHOUSE_EVENTS_FULL_TABLE, - CLICKHOUSE_PENDING_OVERRIDES_FULL_TABLE, - effectiveDistinctIdExpression, - effectivePersonIdExpression, -} from "../../analytics/clickhouse-accessor.ts"; -import { lit, par, type SqlPiece } from "../ir.ts"; -import type { AuthorizedScope } from "../scope.ts"; -import type { - CatalogColumn, - CatalogPropertyNamespace, - CatalogTable, - InjectedScope, - LowerResult, - VoidQLType, -} from "./types.ts"; - -const col = ( - name: string, - type: VoidQLType, - doc: string, - opts: { readonly requires?: readonly ["pii"]; readonly inStar?: boolean } = {}, -): CatalogColumn => ({ - name, - type, - requires: opts.requires ?? [], - inStar: opts.inStar ?? true, - doc, -}); - -/** Columns the inner dedup scan reads off `events_v2` (physical names). */ -const EVENT_INNER_COLUMNS = - "event_id, event_name, event_ts, project_id, distinct_id, person_id, event_properties, context"; - -/** Outer projection re-aliasing physical → logical names, with identity resolution. */ -const eventOuterProjection = (extra: string): string => - `events.event_id AS event_id, events.event_name AS event_name, events.event_ts AS event_ts, ` + - `events.project_id AS project_id, ${effectivePersonIdExpression} AS person_id, ` + - `${effectiveDistinctIdExpression} AS distinct_id, events.event_properties AS event_properties, ` + - `events.context AS context${extra}`; - -/** - * The identity-resolution LEFT JOIN, **scoped inline** (§9, §20). The audited - * `RESOLVED_EVENTS_JOIN` from `clickhouse-accessor.ts` filters the pending-overrides - * read only by `project_id` because it relied on the readonly user's row policy to - * supply the `organization_id` predicate. VoidQL runs under the policy-less - * `analytics_query` user, so we inline `organization_id = {pOrg} AND project_id IN - * {pPids}` into the pending-overrides scan too — otherwise it would read every - * tenant's overrides (the P1-blocker class, here applied to the identity join). The - * inner alias stays `events`/`pending_overrides` so the imported `effective*Id` - * expressions resolve. - */ -const scopedIdentityJoin = (scope: AuthorizedScope): SqlPiece[] => [ - lit( - ` ) AS events LEFT JOIN ( SELECT project_id, source_distinct_id, target_distinct_id, person_id ` + - `FROM ( SELECT project_id, source_distinct_id, target_distinct_id, person_id, is_deleted, version, changed_at ` + - `FROM ${CLICKHOUSE_PENDING_OVERRIDES_FULL_TABLE} WHERE version > 0 AND organization_id = `, - ), - par("String", scope.organizationId), - lit(" AND project_id IN "), - par("Array(String)", scope.availableProjectIds), - lit( - " ORDER BY project_id ASC, source_distinct_id ASC, version DESC, changed_at DESC " + - "LIMIT 1 BY project_id, source_distinct_id ) WHERE is_deleted = 0 ) AS pending_overrides " + - "ON pending_overrides.project_id = events.project_id " + - "AND pending_overrides.source_distinct_id = events.distinct_id", - ), -]; - -/** - * Build the scoped events relation shared by `events` and `revenue`. Emits - * `( FROM ( ) AS events ) AS `, - * with `organization_id`/`project_id` bound out-of-band inside BOTH the dedup scan - * and the identity-join subquery (two scoped physical reads per lowering). - */ -export const buildEventsLower = ( - relation: "events" | "revenue", - scope: AuthorizedScope, - alias: string, - options: { readonly eventNameFilter?: string; readonly extraProjection?: string } = {}, -): LowerResult => { - const pieces: SqlPiece[] = [ - lit( - `( SELECT ${eventOuterProjection(options.extraProjection ?? "")} ` + - `FROM ( SELECT ${EVENT_INNER_COLUMNS} FROM ${CLICKHOUSE_EVENTS_FULL_TABLE} ` + - `WHERE organization_id = `, - ), - par("String", scope.organizationId), - lit(" AND project_id IN "), - par("Array(String)", scope.availableProjectIds), - ]; - if (options.eventNameFilter) { - pieces.push(lit(` AND ${options.eventNameFilter}`)); - } - pieces.push(lit(" ORDER BY processed_ts DESC LIMIT 1 BY event_id")); - pieces.push(...scopedIdentityJoin(scope)); - pieces.push(lit(` ) AS ${alias}`)); - const injected: InjectedScope = { - relation, - alias, - orgValue: scope.organizationId, - projectValues: scope.availableProjectIds, - }; - return { pieces, injected }; -}; - -const namespaces: Readonly> = { - properties: { - name: "properties", - sourceColumn: "event_properties", - requires: [], - doc: "Custom event properties (JSON). Access via properties..", - }, - context: { - name: "context", - sourceColumn: "context", - requires: [], - doc: "Device/SDK/page context captured with the event (JSON). Access via context..", - }, -}; - -export const eventsTable: CatalogTable = { - name: "events", - columns: { - event_id: col("event_id", "String", "Unique id of the event."), - event_name: col("event_name", "String", "The event name, e.g. $pageview."), - event_ts: col("event_ts", "DateTime", "When the event occurred (UTC)."), - project_id: col("project_id", "String", "The project the event belongs to."), - person_id: col("person_id", "String", "Identity-resolved person id."), - distinct_id: col("distinct_id", "String", "Identity-resolved distinct id."), - }, - namespaces, - lower: (scope, alias) => buildEventsLower("events", scope, alias), -}; diff --git a/packages/core/src/services/voidql/catalog/index.ts b/packages/core/src/services/voidql/catalog/index.ts deleted file mode 100644 index 19e983503..000000000 --- a/packages/core/src/services/voidql/catalog/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -/** - * The VoidQL virtual catalog registry — the *only* relation names in the query - * namespace. A `FROM`/`JOIN` relation must resolve to one of these (or a CTE / - * derived alias) or the compiler hard-fails (TRQL's `validateTable`-throws model, - * §11) — so table functions and `system.*` are structurally unreachable. - */ -import { eventsTable } from "./events.ts"; -import { personsTable } from "./persons.ts"; -import { revenueTable } from "./revenue.ts"; -import type { CatalogTable } from "./types.ts"; - -/** - * Catalog version a saved query is authored against; bumped on any change to the - * exposed surface so `analytics_saved_query.schema_version` stays meaningful (§15). - */ -export const CATALOG_SCHEMA_VERSION = 1; - -export const CATALOG: Readonly> = { - events: eventsTable, - persons: personsTable, - revenue: revenueTable, -}; - -/** Reserved internal aliases the substitution injects; not bindable by users (§9). */ -export const RESERVED_INTERNAL_ALIASES = new Set([ - "events", - "persons", - "pending_overrides", - "voidql_union", -]); - -export const getCatalogTable = (name: string): CatalogTable | undefined => CATALOG[name]; - -export * from "./types.ts"; -export { eventsTable, personsTable, revenueTable }; diff --git a/packages/core/src/services/voidql/catalog/persons.ts b/packages/core/src/services/voidql/catalog/persons.ts deleted file mode 100644 index c0e8ce918..000000000 --- a/packages/core/src/services/voidql/catalog/persons.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * The `persons` logical view (§9). Lowers to a latest-version `persons_v1` read - * collapsed with `LIMIT 1 BY (project_id, person_id) ORDER BY version DESC` - * (NOT `row_number() OVER`, which would depend on the deferred window-function - * arm — §18 gap #10), then filtered to `is_archived = 0` on the surviving latest - * row. `email`/`name`/`traits` are PII and require the `pii` capability (§9). - */ -import { CLICKHOUSE_PERSONS_FULL_TABLE } from "../../analytics/clickhouse-accessor.ts"; -import { lit, par, type SqlPiece } from "../ir.ts"; -import type { AuthorizedScope } from "../scope.ts"; -import type { - CatalogColumn, - CatalogPropertyNamespace, - CatalogTable, - InjectedScope, - LowerResult, - VoidQLType, -} from "./types.ts"; - -const col = ( - name: string, - type: VoidQLType, - doc: string, - opts: { readonly requires?: readonly ["pii"]; readonly inStar?: boolean } = {}, -): CatalogColumn => ({ - name, - type, - requires: opts.requires ?? [], - inStar: opts.inStar ?? true, - doc, -}); - -const PERSON_INNER_COLUMNS = - "person_id, project_id, primary_distinct_id, email, name, traits, is_archived"; - -const personOuterProjection = - "person_id AS person_id, project_id AS project_id, primary_distinct_id AS distinct_id, " + - "email AS email, name AS name, traits AS traits"; - -const lower = (scope: AuthorizedScope, alias: string): LowerResult => { - // Inner: latest version per (project_id, person_id). Outer: drop persons whose - // latest row is archived (filter AFTER the collapse, mirroring the overrides - // subquery's `is_deleted` discipline). - const pieces: SqlPiece[] = [ - lit( - `( SELECT ${personOuterProjection} FROM ( SELECT ${PERSON_INNER_COLUMNS} ` + - `FROM ${CLICKHOUSE_PERSONS_FULL_TABLE} WHERE organization_id = `, - ), - par("String", scope.organizationId), - lit(" AND project_id IN "), - par("Array(String)", scope.availableProjectIds), - lit( - ` ORDER BY version DESC LIMIT 1 BY (project_id, person_id) ) WHERE is_archived = 0 ) AS ${alias}`, - ), - ]; - const injected: InjectedScope = { - relation: "persons", - alias, - orgValue: scope.organizationId, - projectValues: scope.availableProjectIds, - }; - return { pieces, injected }; -}; - -const namespaces: Readonly> = { - traits: { - name: "traits", - sourceColumn: "traits", - requires: ["pii"], - doc: "Custom person traits (JSON, PII). Access via traits..", - }, -}; - -export const personsTable: CatalogTable = { - name: "persons", - columns: { - person_id: col("person_id", "String", "Stable person id."), - distinct_id: col("distinct_id", "String", "The person's primary distinct id."), - project_id: col("project_id", "String", "The project the person belongs to."), - email: col("email", "String", "Person email (PII).", { requires: ["pii"], inStar: false }), - name: col("name", "String", "Person name (PII).", { requires: ["pii"], inStar: false }), - }, - namespaces, - lower, -}; diff --git a/packages/core/src/services/voidql/catalog/revenue.ts b/packages/core/src/services/voidql/catalog/revenue.ts deleted file mode 100644 index 84c928c44..000000000 --- a/packages/core/src/services/voidql/catalog/revenue.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * The `revenue` logical view (§9): a semantic view over `events` pre-filtered to - * the revenue event-name set, with `amount_usd` already in dollars and FX-less - * rows zeroed — encoding the accessor's exact money semantics (`amount_usd` is - * stored in cents; an FX-less row contributes 0, a bounded under-count). Agents - * get a "revenue table" without knowing the `$purchase.completed`/`$subscription.*` - * taxonomy. - */ -import { constant } from "@voidhash/lib/lang"; - -import { buildEventsLower, eventsTable } from "./events.ts"; -import type { CatalogColumn, CatalogTable } from "./types.ts"; - -const REVENUE_EVENT_NAMES = constant([ - "$purchase.completed", - "$subscription.created", - "$subscription.renewed", -]); - -// Code-derived constants only (event names + JSON keys) — safe to splice. -const EVENT_NAME_FILTER = `event_name IN (${REVENUE_EVENT_NAMES.map((n) => `'${n}'`).join(", ")})`; - -// USD-only, in dollars: cents (coalesced over snake/camel keys, FX-less → 0) / 100. -const AMOUNT_USD_EXPR = - "(coalesce(nullIf(JSONExtractFloat(events.event_properties, 'amount_usd'), 0), " + - "nullIf(JSONExtractFloat(events.event_properties, 'amountUsd'), 0), 0)) / 100"; - -const amountColumn: CatalogColumn = { - name: "amount_usd", - type: "Float64", - requires: [], - inStar: true, - doc: "Revenue amount in USD dollars (FX-less rows count as 0).", -}; - -export const revenueTable: CatalogTable = { - name: "revenue", - columns: { ...eventsTable.columns, amount_usd: amountColumn }, - namespaces: eventsTable.namespaces, - lower: (scope, alias) => - buildEventsLower("revenue", scope, alias, { - eventNameFilter: EVENT_NAME_FILTER, - extraProjection: `, ${AMOUNT_USD_EXPR} AS amount_usd`, - }), -}; diff --git a/packages/core/src/services/voidql/catalog/types.ts b/packages/core/src/services/voidql/catalog/types.ts deleted file mode 100644 index 9e6c8371a..000000000 --- a/packages/core/src/services/voidql/catalog/types.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** - * The virtual-schema catalog model (§9). Users write SQL against *logical* tables - * (`events`, `persons`, `revenue`) and logical columns — never physical reality. - * A name either resolves to a code-curated, capability-checked physical expression - * or the query is rejected (default-deny; the HogQL virtual-DB model + the - * PSA-2025-00001 lesson). The raw `events_v2`/`persons_v1` tables, the JSON blobs, - * and `organization_id`/`token`/`processed_ts` are simply absent → unresolvable. - */ -import type { SqlPiece } from "../ir.ts"; -import type { AuthorizedScope } from "../scope.ts"; - -export type VoidQLType = "String" | "Int64" | "UInt64" | "Float64" | "Bool" | "DateTime" | "UUID"; - -/** Capabilities resolved once per query from `AuthSession.permissions` + principal kind. */ -export type Capability = "pii"; - -/** An output column of a (sub)query: name + resolved type — drives result decoding. */ -export interface ColumnSpec { - readonly name: string; - readonly type: VoidQLType; -} - -export interface CatalogColumn { - /** Logical name the user types. */ - readonly name: string; - readonly type: VoidQLType; - /** Required capabilities; `[]` = open. */ - readonly requires: readonly Capability[]; - /** Included in `SELECT *`? PII columns are `false` even for a capable caller. */ - readonly inStar: boolean; - /** Surfaced to AI agents and the editor as schema context. */ - readonly doc: string; -} - -/** - * A JSON-blob namespace (`properties`, `context`, `traits`). A `.` - * access lowers to `JSONExtractString(., {pKey:String})` — - * the key is **bound**, never escaped (§9, §18 gap #1). - */ -export interface CatalogPropertyNamespace { - readonly name: string; - /** The logical column on this table holding the JSON String. */ - readonly sourceColumn: string; - readonly requires: readonly Capability[]; - readonly doc: string; -} - -/** Recorded per base-table occurrence; the verifier checks each binds to the scope. */ -export interface InjectedScope { - readonly relation: "events" | "persons" | "revenue"; - readonly alias: string; - readonly orgValue: string; - readonly projectValues: readonly string[]; -} - -export interface LowerResult { - /** The `(…scoped subquery…) AS ` relation pieces. */ - readonly pieces: readonly SqlPiece[]; - /** The scope this lowering injected, for the value-level verifier. */ - readonly injected: InjectedScope; -} - -export interface CatalogTable { - readonly name: "events" | "persons" | "revenue"; - readonly columns: Readonly>; - readonly namespaces: Readonly>; - /** - * Lower this logical view to a pre-scoped subquery carrying the bound - * `organization_id`/`project_id` predicate inline (NOT via `tenantSettings` — - * the P1 blocker), wrapping the audited dedup/identity machinery. The ONLY - * producer of physical table identity. - */ - readonly lower: (scope: AuthorizedScope, alias: string) => LowerResult; -} - -/** Map a logical type to the explicit ClickHouse `ch.param` type string (§18 gap #9). */ -export const chParamType = (type: VoidQLType): string => { - switch (type) { - case "String": - case "UUID": - return "String"; - case "Int64": - return "Int64"; - case "UInt64": - return "UInt64"; - case "Float64": - return "Float64"; - case "Bool": - return "Bool"; - case "DateTime": - return "DateTime"; - } -}; diff --git a/packages/core/src/services/voidql/compile.ts b/packages/core/src/services/voidql/compile.ts deleted file mode 100644 index f6420d197..000000000 --- a/packages/core/src/services/voidql/compile.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * The VoidQL compile orchestration — the pure stages ①–⑥ of the VM (§12): - * `tokenize → parse → resolve+print → verify`, yielding a {@link CompiledQuery} - * that the service executes. No ClickHouse, no Db, no Auth here — trivially - * unit-testable and fuzzable. - */ -import { createId } from "@paralleldrive/cuid2"; -import { Effect } from "effect"; - -import type { Capability, ColumnSpec } from "./catalog/types.ts"; -import { type CompiledSelect, compileSelect } from "./compiler.ts"; -import { isVoidQlCompileError, type VoidQlCompileError } from "./errors.ts"; -import type { SqlPiece } from "./ir.ts"; -import { parse } from "./parser.ts"; -import type { AuthorizedScope } from "./scope.ts"; -import { verify } from "./verify.ts"; - -export interface CompiledQuery { - readonly pieces: readonly SqlPiece[]; - /** Output column names + types — used to decode the result set (§20). */ - readonly columns: readonly ColumnSpec[]; - /** Server-random ClickHouse query id (the KILL handle; never user input, §18 #11). */ - readonly queryId: string; -} - -const makeQueryId = (): string => globalThis.crypto?.randomUUID?.() ?? `voidql-${createId()}`; - -/** - * Pure `parse → resolve+print` (no verify). Throws the typed compile errors. - * Exposed for unit tests and the validate/repair loop. - */ -export const compileToIr = ( - text: string, - scope: AuthorizedScope, - capabilities: ReadonlySet, -): CompiledSelect => compileSelect(parse(text), scope, capabilities); - -/** - * Full pure pipeline including the value-level verifier. Throws on the compile-error - * union or an `VoidQlIsolationError` defect. - */ -export const compilePure = ( - text: string, - scope: AuthorizedScope, - capabilities: ReadonlySet, -): CompiledQuery => { - const compiled = compileToIr(text, scope, capabilities); - verify(compiled.pieces, compiled.injected, scope); - return { pieces: compiled.pieces, columns: compiled.shape, queryId: makeQueryId() }; -}; - -/** - * Compile VoidQL text to a {@link CompiledQuery} as an Effect. Typed compile errors - * surface in the error channel; any non-VoidQL throwable becomes a defect (a real - * bug), never a silent failure. - */ -export const compileVoidQl = ( - text: string, - scope: AuthorizedScope, - capabilities: ReadonlySet, -): Effect.Effect => - Effect.suspend(() => - Effect.try({ - try: () => compilePure(text, scope, capabilities), - catch: (error) => error, - }).pipe( - Effect.catch((error) => { - if (isVoidQlCompileError(error)) return Effect.fail(error); - return Effect.die(error); - }), - ), - ); diff --git a/packages/core/src/services/voidql/compiler.ts b/packages/core/src/services/voidql/compiler.ts deleted file mode 100644 index a33c855fc..000000000 --- a/packages/core/src/services/voidql/compiler.ts +++ /dev/null @@ -1,1027 +0,0 @@ -/* - * This module is the VoidQL resolve-and-print pass: a single recursive walk over the - * AST where `throw` IS the control flow. Every one of the ~32 throw sites aborts the - * walk with a typed compile error (VoidQlUnsupportedError / VoidQlSchemaError / - * VoidQlUnknownFieldError / VoidQlPiiError / VoidQlSyntaxError) from deep inside a - * mutually-recursive printer whose methods return `SqlPiece[]`/`string`, not Effects. - * `compile.ts` is the single Effect boundary for this stage and converts these throws - * into the typed `VoidQlCompileError` union (see `isVoidQlCompileError`); threading - * Effect through the printer instead would surface those typed compile errors as - * opaque defects and rewrite the whole pass. - */ -// oxlint-disable effect/noThrowStatement -- throw is the deliberate control flow of this pure printer; compile.ts is the single Effect boundary that converts these typed compile errors (see block comment above). -/** - * The VoidQL resolve-and-print pass — stages ③–⑤ of the VM (§12). - * - * Walks the AST with a scope stack, resolving every name against the virtual - * catalog, the closed function registry, and the PII capability set; performs the - * **logical-view substitution** (L2, §10) so each base relation lowers to a - * pre-scoped subquery carrying the bound `organization_id`/`project_id` literals; - * and emits the {@link SqlPiece} IR through `lit()`/`par()` — every user value a - * bound parameter, all structural SQL from frozen catalog constants and validated - * identifiers. The result feeds the value-level verifier (§10) and {@link toStatement}. - * - * Resolution and printing are a single pass because the alias a column binds to is - * needed exactly at print time (HogQL/TRQL both resolve-while-printing). Field-level - * re-assertions the dropped decode boundary used to do (a non-empty `chain`, a valid - * `numType`) live here. - */ -import { DateTime, Option } from "effect"; - -import { toClickhouseDateTime } from "../analytics/clickhouse-accessor.ts"; -import type { - Binary, - CaseExpr, - ColumnRef, - Expr, - FnCall, - InExpr, - Join, - OrderItem, - Query, - Select, - SelectItem, - TableSource, -} from "./ast/VoidQlAst.ts"; -import { CATALOG, RESERVED_INTERNAL_ALIASES } from "./catalog/index.ts"; -import { - type CatalogTable, - type Capability, - chParamType, - type ColumnSpec, - type VoidQLType, -} from "./catalog/types.ts"; -import { - VoidQlPiiError, - VoidQlSchemaError, - VoidQlSyntaxError, - VoidQlUnknownFieldError, - VoidQlUnsupportedError, -} from "./errors.ts"; -import { type FnSpec, lookupFunction } from "./functions.ts"; -import type { InjectedScope } from "./catalog/types.ts"; -import { lit, par, type SqlPiece } from "./ir.ts"; -import type { AuthorizedScope } from "./scope.ts"; - -/** Largest result set VoidQL will return; the server clamps every LIMIT to it (§7 L6, §12). */ -export const MAX_RESULT_ROWS = 100_000; - -const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; -// Identifiers shaped like a physical table version suffix (`…_v2`). Reserved as -// non-bindable so the value-level verifier's physical-table token scan can never -// be tripped by a legitimate user alias / CTE / column alias (a false-positive -// isolation failure). Physical tables are emitted only by the catalog `lower()`. -const PHYSICAL_SHAPED_RE = /_v\d+$/i; - -type ResolvedRelation = - | { readonly kind: "view"; readonly alias: string; readonly table: CatalogTable } - | { - readonly kind: "derived"; - readonly alias: string; - readonly columns: ReadonlyMap; - }; - -interface Frame { - readonly relations: ResolvedRelation[]; - readonly cteShapes: Map; - readonly aliases: Map; - readonly parent?: Frame; -} - -export interface CompiledSelect { - readonly pieces: readonly SqlPiece[]; - /** Output columns (name + type) — drives result decoding (§20 open question). */ - readonly shape: readonly ColumnSpec[]; - /** Every base-table scope this compilation injected (for the verifier). */ - readonly injected: readonly InjectedScope[]; -} - -interface PrintedExpr { - readonly pieces: readonly SqlPiece[]; - readonly type: VoidQLType; -} - -const substitutionCost = (left: string | undefined, right: string | undefined): number => { - if (left === right) return 0; - return 1; -}; - -const levenshtein = (a: string, b: string): number => { - const m = a.length; - const n = b.length; - const dp = Array.from({ length: m + 1 }, () => Array.from({ length: n + 1 }, () => 0)); - for (let i = 0; i <= m; i++) dp[i]![0] = i; - for (let j = 0; j <= n; j++) dp[0]![j] = j; - for (let i = 1; i <= m; i++) { - for (let j = 1; j <= n; j++) { - const cost = substitutionCost(a[i - 1], b[j - 1]); - dp[i]![j] = Math.min(dp[i - 1]![j]! + 1, dp[i]![j - 1]! + 1, dp[i - 1]![j - 1]! + cost); - } - } - return dp[m]![n]!; -}; - -/** `ASC`/`DESC` for an ORDER BY item — shared by plain and window ORDER BY printing. */ -const orderDirectionSql = (dir: OrderItem["dir"]): string => { - if (dir === "desc") return " DESC"; - return " ASC"; -}; - -/** `NULLS FIRST`/`NULLS LAST` for an ORDER BY item. */ -const orderNullsSql = (nulls: "first" | "last"): string => { - if (nulls === "first") return " NULLS FIRST"; - return " NULLS LAST"; -}; - -const joinKindSql = (kind: Join["kind"]): string => { - if (kind === "left") return " LEFT JOIN "; - if (kind === "right") return " RIGHT JOIN "; - if (kind === "full") return " FULL JOIN "; - if (kind === "cross") return " CROSS JOIN "; - return " INNER JOIN "; -}; - -/** A numeric literal adopts the expected column width when one is propagated. */ -const numberLitType = (expected: VoidQLType | undefined, declared: VoidQLType): VoidQLType => { - if (expected === "Int64" || expected === "UInt64" || expected === "Float64") return expected; - return declared; -}; - -export type { ColumnSpec }; - -const nearest = (target: string, candidates: readonly string[]): string => { - let best = ""; - let bestDist = Infinity; - for (const c of candidates) { - const d = levenshtein(target.toLowerCase(), c.toLowerCase()); - if (d < bestDist) { - bestDist = d; - best = c; - } - } - if (bestDist <= Math.max(2, Math.ceil(target.length / 2))) return best; - return ""; -}; - -export class Compiler { - readonly injected: InjectedScope[] = []; - private relationCounter = 0; - - private readonly scope: AuthorizedScope; - private readonly capabilities: ReadonlySet; - - constructor(scope: AuthorizedScope, capabilities: ReadonlySet) { - this.scope = scope; - this.capabilities = capabilities; - } - - // ── relations / sources ─────────────────────────────────────────────────── - - private assertBindableAlias(alias: string): void { - if (!IDENT_RE.test(alias) || PHYSICAL_SHAPED_RE.test(alias)) { - throw new VoidQlUnsupportedError({ message: `Invalid alias '${alias}'.`, hint: "" }); - } - if (RESERVED_INTERNAL_ALIASES.has(alias.toLowerCase())) { - throw new VoidQlUnsupportedError({ - message: `'${alias}' is a reserved alias and cannot be used.`, - hint: "Pick a different alias.", - }); - } - } - - private viewShape(table: CatalogTable): ReadonlyMap { - return new Map(Object.values(table.columns).map((c) => [c.name, c.type])); - } - - /** Resolve a FROM/JOIN source, push it onto `frame`, and return its SQL pieces. */ - private resolveSource(source: TableSource, frame: Frame): readonly SqlPiece[] { - if (source._tag === "SubquerySource") { - this.assertBindableAlias(source.alias); - this.assertUniqueAlias(frame, source.alias); - const sub = this.printQuery(source.query, frame); - frame.relations.push({ - kind: "derived", - alias: source.alias, - columns: new Map(sub.shape.map((c) => [c.name, c.type])), - }); - return [lit("( "), ...sub.pieces, lit(` ) AS ${source.alias}`)]; - } - - const nameLower = source.name.toLowerCase(); - const cteShape = this.findCteShape(frame, nameLower); - if (cteShape) { - const alias = source.alias ?? source.name; - if (source.alias) this.assertBindableAlias(source.alias); - this.assertUniqueAlias(frame, alias); - frame.relations.push({ - kind: "derived", - alias, - columns: new Map(cteShape.map((c) => [c.name, c.type])), - }); - let tail = ""; - if (source.alias) tail = ` AS ${this.id(source.alias)}`; - return [lit(`${this.id(source.name)}${tail}`)]; - } - - const table = CATALOG[nameLower]; - if (!table) { - const suggestion = nearest(source.name, Object.keys(CATALOG)); - let didYouMean = ""; - if (suggestion) didYouMean = ` Did you mean '${suggestion}'?`; - throw new VoidQlSchemaError({ - message: `Unknown table '${source.name}'.${didYouMean}`, - }); - } - if (source.alias) this.assertBindableAlias(source.alias); - const alias = source.alias ?? `${table.name}_${this.relationCounter++}`; - this.assertUniqueAlias(frame, alias); - const lowered = table.lower(this.scope, alias); - this.injected.push(lowered.injected); - frame.relations.push({ kind: "view", alias, table }); - return lowered.pieces; - } - - private assertUniqueAlias(frame: Frame, alias: string): void { - if (frame.relations.some((r) => r.alias.toLowerCase() === alias.toLowerCase())) { - throw new VoidQlUnsupportedError({ - message: `Duplicate relation alias '${alias}'.`, - hint: "Give each table/subquery a distinct alias.", - }); - } - } - - private findRelation(frame: Frame, aliasLower: string): ResolvedRelation | undefined { - return frame.relations.find((relation) => relation.alias.toLowerCase() === aliasLower); - } - - private findExpressionAlias(frame: Frame, aliasLower: string): VoidQLType | undefined { - return frame.aliases.get(aliasLower); - } - - private findCteShape( - frame: Frame | undefined, - nameLower: string, - ): readonly ColumnSpec[] | undefined { - for (let f = frame; f; f = f.parent) { - const match = f.cteShapes.get(nameLower); - if (match) return match; - } - return undefined; - } - - // ── columns / properties ──────────────────────────────────────────────── - - private relationColumnNames(rel: ResolvedRelation): readonly string[] { - if (rel.kind === "view") return Object.keys(rel.table.columns); - return [...rel.columns.keys()]; - } - - /** - * Validate a single identifier at the point it is spliced into `lit()` — making - * "only charset-safe identifiers reach ch.literal" explicit at the emission site - * rather than an upstream invariant (§12). Every alias/column/namespace identifier - * is already catalog-derived or parser-tokenised, so this never fires in practice; - * it is the belt against a future code path that forgets to validate. - */ - private id(name: string): string { - if (!IDENT_RE.test(name)) { - throw new VoidQlUnsupportedError({ message: `Invalid identifier '${name}'.`, hint: "" }); - } - return name; - } - - /** A validated `.` reference. */ - private qualified(alias: string, column: string): string { - return `${this.id(alias)}.${this.id(column)}`; - } - - private piiGate(requires: readonly Capability[], what: string): void { - if (requires.includes("pii") && !this.capabilities.has("pii")) { - throw new VoidQlPiiError({ - message: `${what} is PII and requires elevated access; the query was rejected.`, - }); - } - } - - /** Lower a JSON-namespace property access; the key is bound, never spliced (§9). */ - private property(alias: string, sourceColumn: string, key: string): PrintedExpr { - return { - pieces: [ - lit(`JSONExtractString(${this.qualified(alias, sourceColumn)}, `), - par("String", key), - lit(")"), - ], - type: "String", - }; - } - - private columnOf(rel: ResolvedRelation, name: string): VoidQLType | undefined { - if (rel.kind === "view") { - const col = rel.table.columns[name]; - if (!col) return undefined; - this.piiGate(col.requires, `Column '${name}'`); - return col.type; - } - return rel.columns.get(name); - } - - private resolveColumn(ref: ColumnRef, frame: Frame): PrintedExpr { - const chain = ref.chain; - if (chain.length > 3) { - throw new VoidQlUnsupportedError({ - message: `Property path '${chain.join(".")}' is too deep.`, - hint: "", - }); - } - - // qualified namespace access: .. - if (chain.length === 3) { - const rel = this.findRelation(frame, chain[0].toLowerCase()); - if (!rel || rel.kind !== "view") { - throw new VoidQlUnknownFieldError({ - field: chain.join("."), - message: `Unknown property path '${chain.join(".")}'.`, - suggestion: "", - }); - } - const ns = rel.table.namespaces[chain[1]]; - if (!ns) { - throw new VoidQlUnknownFieldError({ - field: `${chain[0]}.${chain[1]}`, - message: `'${chain[1]}' is not a property namespace on '${rel.table.name}'.`, - suggestion: nearest(chain[1], Object.keys(rel.table.namespaces)), - }); - } - this.piiGate(ns.requires, `Property namespace '${ns.name}'`); - return this.property(rel.alias, ns.sourceColumn, chain[2]); - } - - if (chain.length === 2) { - const [a, b] = chain; - const rel = this.findRelation(frame, a.toLowerCase()); - if (rel) { - // . - const type = this.columnOf(rel, b); - if (type) return { pieces: [lit(this.qualified(rel.alias, b))], type }; - if (rel.kind === "view" && rel.table.namespaces[b]) { - throw new VoidQlUnknownFieldError({ - field: `${a}.${b}`, - message: `'${b}' is a property namespace; specify a key, e.g. ${b}..`, - suggestion: "", - }); - } - throw new VoidQlUnknownFieldError({ - field: `${a}.${b}`, - message: `Unknown column '${b}' on '${a}'.`, - suggestion: nearest(b, this.relationColumnNames(rel)), - }); - } - // unqualified namespace access: . - const nsRel = this.findNamespaceRelation(frame, a); - if (nsRel) { - const ns = nsRel.table.namespaces[a]!; - this.piiGate(ns.requires, `Property namespace '${ns.name}'`); - return this.property(nsRel.alias, ns.sourceColumn, b); - } - throw new VoidQlUnknownFieldError({ - field: `${a}.${b}`, - message: `Unknown reference '${a}.${b}'.`, - suggestion: "", - }); - } - - // bare column name - const name = chain[0]; - const matches = this.currentRelations(frame).filter((r) => - this.relationColumnNames(r).includes(name), - ); - if (matches.length === 1) { - const type = this.columnOf(matches[0]!, name)!; - return { pieces: [lit(this.qualified(matches[0]!.alias, name))], type }; - } - if (matches.length > 1) { - throw new VoidQlUnknownFieldError({ - field: name, - message: `Column '${name}' is ambiguous; qualify it with a table alias.`, - suggestion: "", - }); - } - const aliasType = this.findExpressionAlias(frame, name.toLowerCase()); - if (aliasType) { - return { pieces: [lit(this.id(name))], type: aliasType }; - } - const allCols = this.currentRelations(frame).flatMap((r) => this.relationColumnNames(r)); - if (this.findNamespaceRelation(frame, name)) { - throw new VoidQlUnknownFieldError({ - field: name, - message: `'${name}' is a property namespace; specify a key, e.g. ${name}..`, - suggestion: "", - }); - } - throw new VoidQlUnknownFieldError({ - field: name, - message: `Unknown column '${name}'.`, - suggestion: nearest(name, allCols), - }); - } - - private currentRelations(frame: Frame): readonly ResolvedRelation[] { - return frame.relations; - } - - private findNamespaceRelation( - frame: Frame, - name: string, - ): { readonly alias: string; readonly table: CatalogTable } | undefined { - const hits = frame.relations.filter( - (r): r is Extract => - r.kind === "view" && name in r.table.namespaces, - ); - if (hits.length !== 1) return undefined; - return { alias: hits[0]!.alias, table: hits[0]!.table }; - } - - // ── expressions ──────────────────────────────────────────────────────────── - - private printExpr(expr: Expr, frame: Frame, expected?: VoidQLType): PrintedExpr { - switch (expr._tag) { - case "StringLit": { - if (expected !== "DateTime") { - return { pieces: [par("String", expr.value)], type: "String" }; - } - // A string compared against a DateTime column is coerced to DateTime for - // partition pruning. Guard the parse: an unparseable literal would otherwise - // escape as an opaque defect/500 and break the validate-repair loop (§18 #9). - // `Date.parse` (not `DateTime.make(string)`) keeps the host's literal-parsing - // semantics — `DateTime.make` appends `Z` to zone-less strings. - const parsed = DateTime.make(Date.parse(expr.value)); - if (Option.isNone(parsed)) { - throw new VoidQlSyntaxError({ - message: `'${expr.value}' is not a valid date/time literal.`, - hint: "Use an ISO-8601 date like '2026-01-01' or '2026-01-01 12:00:00'.", - }); - } - return { - pieces: [par("DateTime", toClickhouseDateTime(DateTime.toDateUtc(parsed.value)))], - type: "DateTime", - }; - } - case "NumberLit": { - const type = numberLitType(expected, expr.numType); - return { pieces: [par(chParamType(type), expr.value)], type }; - } - case "BoolLit": - return { pieces: [par("Bool", expr.value)], type: "Bool" }; - case "NullLit": - return { pieces: [lit("NULL")], type: expected ?? "String" }; - case "ColumnRef": - return this.resolveColumn(expr, frame); - case "StarRef": - throw new VoidQlUnsupportedError({ - message: "'*' is only allowed as a SELECT item or as count(*).", - hint: "", - }); - case "Paren": { - const inner = this.printExpr(expr.expr, frame, expected); - return { pieces: [lit("("), ...inner.pieces, lit(")")], type: inner.type }; - } - case "Unary": { - let innerExpected: VoidQLType | undefined = undefined; - if (expr.op === "neg") innerExpected = expected; - const inner = this.printExpr(expr.expr, frame, innerExpected); - if (expr.op === "not") { - return { pieces: [lit("(NOT "), ...inner.pieces, lit(")")], type: "Bool" }; - } - return { pieces: [lit("(-"), ...inner.pieces, lit(")")], type: inner.type }; - } - case "Binary": - return this.printBinary(expr, frame); - case "InExpr": - return this.printIn(expr, frame); - case "ExistsExpr": { - const query = this.printQuery(expr.query, frame); - return { pieces: [lit("EXISTS ( "), ...query.pieces, lit(" )")], type: "Bool" }; - } - case "SubqueryExpr": { - const query = this.printQuery(expr.query, frame); - if (query.shape.length !== 1) { - throw new VoidQlUnsupportedError({ - message: "A scalar subquery must return exactly one column.", - hint: "Select one expression in the scalar subquery.", - }); - } - return { - pieces: [lit("( "), ...query.pieces, lit(" )")], - type: query.shape[0]!.type, - }; - } - case "Between": { - const target = this.printExpr(expr.expr, frame); - const low = this.printExpr(expr.low, frame, target.type); - const high = this.printExpr(expr.high, frame, target.type); - let betweenOp = " BETWEEN "; - if (expr.negated) betweenOp = " NOT BETWEEN "; - return { - pieces: [ - lit("("), - ...target.pieces, - lit(betweenOp), - ...low.pieces, - lit(" AND "), - ...high.pieces, - lit(")"), - ], - type: "Bool", - }; - } - case "IsNull": { - const target = this.printExpr(expr.expr, frame); - let nullTest = " IS NULL)"; - if (expr.negated) nullTest = " IS NOT NULL)"; - return { - pieces: [lit("("), ...target.pieces, lit(nullTest)], - type: "Bool", - }; - } - case "CaseExpr": - return this.printCase(expr, frame); - case "FnCall": - return this.printFnCall(expr, frame); - case "WindowExpr": { - const fn = this.printFnCall(expr.fn, frame, true); - const pieces: SqlPiece[] = [...fn.pieces, lit(" OVER (")]; - if (expr.partitionBy.length > 0) { - pieces.push(lit("PARTITION BY ")); - expr.partitionBy.forEach((partition, index) => { - if (index > 0) pieces.push(lit(", ")); - pieces.push(...this.printExpr(partition, frame).pieces); - }); - } - if (expr.orderBy.length > 0) { - if (expr.partitionBy.length > 0) pieces.push(lit(" ")); - pieces.push(lit("ORDER BY ")); - expr.orderBy.forEach((order, index) => { - if (index > 0) pieces.push(lit(", ")); - pieces.push( - ...this.printExpr(order.expr, frame).pieces, - lit(orderDirectionSql(order.dir)), - ); - if (order.nulls) { - pieces.push(lit(orderNullsSql(order.nulls))); - } - }); - } - if (expr.frame) { - if (expr.partitionBy.length > 0 || expr.orderBy.length > 0) pieces.push(lit(" ")); - const bound = (value: typeof expr.frame.start): string => { - if (value === "unboundedPreceding") return "UNBOUNDED PRECEDING"; - if (value === "unboundedFollowing") return "UNBOUNDED FOLLOWING"; - return "CURRENT ROW"; - }; - let frameUnitSql = "RANGE "; - if (expr.frame.unit === "rows") frameUnitSql = "ROWS "; - pieces.push(lit(frameUnitSql)); - if (expr.frame.end) { - pieces.push(lit(`BETWEEN ${bound(expr.frame.start)} AND ${bound(expr.frame.end)}`)); - } else { - pieces.push(lit(bound(expr.frame.start))); - } - } - pieces.push(lit(")")); - return { pieces, type: fn.type }; - } - } - } - - private printBinary(expr: Binary, frame: Frame): PrintedExpr { - const sqlOp: Record = { - or: " OR ", - and: " AND ", - eq: " = ", - neq: " != ", - lt: " < ", - lte: " <= ", - gt: " > ", - gte: " >= ", - like: " LIKE ", - notLike: " NOT LIKE ", - ilike: " ILIKE ", - notIlike: " NOT ILIKE ", - add: " + ", - sub: " - ", - mul: " * ", - div: " / ", - mod: " % ", - }; - const isComparison = ["eq", "neq", "lt", "lte", "gt", "gte"].includes(expr.op); - const isLogical = expr.op === "and" || expr.op === "or"; - const isArith = ["add", "sub", "mul", "div", "mod"].includes(expr.op); - const isPattern = - expr.op === "like" || expr.op === "notLike" || expr.op === "ilike" || expr.op === "notIlike"; - - const left = this.printExpr(expr.left, frame); - // Propagate the left operand's type to a comparison RHS so a string literal - // compared to a DateTime column binds as DateTime (partition pruning, §18 #9). - let rhsExpected: VoidQLType | undefined = undefined; - if (isComparison) rhsExpected = left.type; - else if (isPattern) rhsExpected = "String"; - const right = this.printExpr(expr.right, frame, rhsExpected); - let type: VoidQLType = "String"; - if (isComparison || isLogical || isPattern) type = "Bool"; - else if (isArith) type = "Float64"; - return { - pieces: [lit("("), ...left.pieces, lit(sqlOp[expr.op]), ...right.pieces, lit(")")], - type, - }; - } - - private printIn(expr: InExpr, frame: Frame): PrintedExpr { - const target = this.printExpr(expr.expr, frame); - let inOp = " IN ("; - if (expr.negated) inOp = " NOT IN ("; - const pieces: SqlPiece[] = [lit("("), ...target.pieces, lit(inOp)]; - if (expr.query) { - const query = this.printQuery(expr.query, frame); - if (query.shape.length !== 1) { - throw new VoidQlUnsupportedError({ - message: "An IN subquery must return exactly one column.", - hint: "Select one expression in the IN subquery.", - }); - } - pieces.push(...query.pieces); - } else { - expr.list?.forEach((item, i) => { - if (i > 0) pieces.push(lit(", ")); - pieces.push(...this.printExpr(item, frame, target.type).pieces); - }); - } - pieces.push(lit("))")); - return { pieces, type: "Bool" }; - } - - private printCase(expr: CaseExpr, frame: Frame): PrintedExpr { - const pieces: SqlPiece[] = [lit("CASE")]; - if (expr.operand) { - pieces.push(lit(" "), ...this.printExpr(expr.operand, frame).pieces); - } - let resultType: VoidQLType = "String"; - expr.whens.forEach((branch, i) => { - const when = this.printExpr(branch.when, frame); - const then = this.printExpr(branch.then, frame); - if (i === 0) resultType = then.type; - pieces.push(lit(" WHEN "), ...when.pieces, lit(" THEN "), ...then.pieces); - }); - if (expr.else) { - pieces.push(lit(" ELSE "), ...this.printExpr(expr.else, frame).pieces); - } - pieces.push(lit(" END")); - return { pieces, type: resultType }; - } - - private printFnCall(expr: FnCall, frame: Frame, inWindow = false): PrintedExpr { - const spec = lookupFunction(expr.name); - if (!spec) { - throw new VoidQlUnsupportedError({ - message: `Unknown or unsupported function '${expr.name}'.`, - hint: "Only the curated VoidQL function set is available.", - }); - } - if (expr.args.length < spec.minArgs || expr.args.length > spec.maxArgs) { - throw new VoidQlUnsupportedError({ - message: `Function '${expr.name}' expects ${spec.minArgs}..${spec.maxArgs} arguments, got ${expr.args.length}.`, - hint: "", - }); - } - if (spec.windowOnly && !inWindow) { - throw new VoidQlUnsupportedError({ - message: `Function '${expr.name}' requires an OVER clause.`, - hint: "Add OVER (...) to make this a window expression.", - }); - } - - // count(*) is the only place a bare star is admissible. - if (spec.allowStar && expr.args.length === 1 && expr.args[0]!._tag === "StarRef") { - return { pieces: [lit(`${spec.chName}(*)`)], type: this.fnReturnType(spec, []) }; - } - - const argResults = expr.args.map((arg) => { - if (arg._tag === "StarRef") { - throw new VoidQlUnsupportedError({ - message: `'*' is not a valid argument to '${expr.name}'.`, - hint: "", - }); - } - return this.printExpr(arg, frame); - }); - - const pieces: SqlPiece[] = [lit(`${spec.chName}(`)]; - argResults.forEach((arg, i) => { - if (i > 0) pieces.push(lit(", ")); - pieces.push(...arg.pieces); - }); - pieces.push(lit(")")); - return { - pieces, - type: this.fnReturnType( - spec, - argResults.map((a) => a.type), - ), - }; - } - - private fnReturnType(spec: FnSpec, argTypes: readonly VoidQLType[]): VoidQLType { - if (typeof spec.returns === "object") { - return argTypes[spec.returns.arg] ?? "String"; - } - return spec.returns; - } - - // ── select ────────────────────────────────────────────────────────────── - - printQuery(query: Query, parent?: Frame): CompiledSelect { - if (query._tag === "Select") return this.printSelect(query, parent); - - const compiled = query.selects.map((select) => this.printSelect(select, parent)); - const shape = compiled[0]!.shape; - for (const arm of compiled.slice(1)) { - const compatible = - arm.shape.length === shape.length && - arm.shape.every((column, index) => column.type === shape[index]!.type); - if (!compatible) { - throw new VoidQlUnsupportedError({ - message: "Set-query arms must return the same number of columns with matching types.", - hint: "Align each SELECT projection before combining them.", - }); - } - } - const unionPieces: SqlPiece[] = []; - compiled.forEach((arm, index) => { - if (index > 0) unionPieces.push(lit(` ${query.operators[index - 1]} `)); - unionPieces.push(...arm.pieces); - }); - const pieces: SqlPiece[] = [lit("SELECT ")]; - shape.forEach((column, index) => { - if (index > 0) pieces.push(lit(", ")); - pieces.push(lit(`${this.qualified("voidql_union", column.name)} AS ${this.id(column.name)}`)); - }); - pieces.push( - lit(" FROM ( "), - ...unionPieces, - lit(` ) AS voidql_union LIMIT ${MAX_RESULT_ROWS}`), - ); - return { pieces, shape, injected: [] }; - } - - printSelect(select: Select, parent?: Frame): CompiledSelect { - const frame: Frame = { relations: [], cteShapes: new Map(), aliases: new Map(), parent }; - - const ctePieces: SqlPiece[] = []; - select.with.forEach((cte, i) => { - const nameLower = cte.name.toLowerCase(); - if ( - !IDENT_RE.test(cte.name) || - PHYSICAL_SHAPED_RE.test(cte.name) || - RESERVED_INTERNAL_ALIASES.has(nameLower) - ) { - throw new VoidQlUnsupportedError({ message: `Invalid CTE name '${cte.name}'.`, hint: "" }); - } - if (frame.cteShapes.has(nameLower)) { - throw new VoidQlUnsupportedError({ message: `Duplicate CTE '${cte.name}'.`, hint: "" }); - } - const compiled = this.printQuery(cte.query, frame); - frame.cteShapes.set(nameLower, compiled.shape); - let cteLead = ", "; - if (i === 0) cteLead = "WITH "; - ctePieces.push( - lit(cteLead), - lit(`${cte.name} AS ( `), - ...compiled.pieces, - lit(" )"), - ); - }); - if (ctePieces.length > 0) ctePieces.push(lit(" ")); - - // FROM + JOINs build the relation frame before any expression is resolved. - let fromPieces: readonly SqlPiece[] = []; - if (select.from) fromPieces = this.resolveSource(select.from, frame); - const joinPieces: SqlPiece[] = []; - for (const join of select.joins) { - const relationCountBeforeJoin = frame.relations.length; - const sourcePieces = this.resolveSource(join.source, frame); - joinPieces.push(lit(joinKindSql(join.kind)), ...sourcePieces); - if (join.on) { - joinPieces.push(lit(" ON "), ...this.printExpr(join.on, frame, "Bool").pieces); - } else if (join.using) { - for (const column of join.using) { - const leftMatches = frame.relations - .slice(0, relationCountBeforeJoin) - .filter((relation) => this.relationColumnNames(relation).includes(column)); - const rightMatches = frame.relations - .slice(relationCountBeforeJoin) - .filter((relation) => this.relationColumnNames(relation).includes(column)); - if (leftMatches.length === 0 || rightMatches.length === 0) { - throw new VoidQlUnknownFieldError({ - field: column, - message: `USING column '${column}' must exist on both sides of the join.`, - suggestion: "", - }); - } - [...leftMatches, ...rightMatches].forEach((relation) => this.columnOf(relation, column)); - } - joinPieces.push(lit(` USING (${join.using.map((column) => this.id(column)).join(", ")})`)); - } - } - - const { columnPieces, shape } = this.printColumns(select.columns, frame); - - let selectKeyword = "SELECT"; - if (select.distinct) selectKeyword = "SELECT DISTINCT"; - const pieces: SqlPiece[] = [...ctePieces, lit(selectKeyword)]; - if (select.distinctOn.length > 0) { - pieces.push(lit(" ON (")); - select.distinctOn.forEach((expr, index) => { - if (index > 0) pieces.push(lit(", ")); - pieces.push(...this.printExpr(expr, frame).pieces); - }); - pieces.push(lit(")")); - } - pieces.push(lit(" "), ...columnPieces); - if (select.from) pieces.push(lit(" FROM "), ...fromPieces); - pieces.push(...joinPieces); - - if (select.prewhere || select.where) { - pieces.push(lit(" WHERE ")); - if (select.prewhere) { - pieces.push(...this.printExpr(select.prewhere, frame, "Bool").pieces); - } - if (select.prewhere && select.where) pieces.push(lit(" AND ")); - if (select.where) pieces.push(...this.printExpr(select.where, frame, "Bool").pieces); - } - if (select.groupBy.length > 0) { - pieces.push(lit(" GROUP BY ")); - select.groupBy.forEach((g, i) => { - if (i > 0) pieces.push(lit(", ")); - pieces.push(...this.printExpr(g, frame).pieces); - }); - if (select.groupByModifier) { - let modifierSql = " WITH CUBE"; - if (select.groupByModifier === "rollup") modifierSql = " WITH ROLLUP"; - pieces.push(lit(modifierSql)); - } - if (select.withTotals) pieces.push(lit(" WITH TOTALS")); - } - if (select.having) { - pieces.push(lit(" HAVING "), ...this.printExpr(select.having, frame, "Bool").pieces); - } - if (select.qualify) { - pieces.push(lit(" QUALIFY "), ...this.printExpr(select.qualify, frame, "Bool").pieces); - } - if (select.orderBy.length > 0) { - pieces.push(lit(" ORDER BY ")); - select.orderBy.forEach((o: OrderItem, i) => { - if (i > 0) pieces.push(lit(", ")); - pieces.push(...this.printExpr(o.expr, frame).pieces, lit(orderDirectionSql(o.dir))); - if (o.nulls) pieces.push(lit(orderNullsSql(o.nulls))); - }); - } - - if (select.limitBy) { - pieces.push(lit(` LIMIT ${select.limitBy.limit}`)); - if (select.limitBy.offset && select.limitBy.offset > 0) { - pieces.push(lit(` OFFSET ${select.limitBy.offset}`)); - } - pieces.push(lit(" BY ")); - select.limitBy.by.forEach((expr, index) => { - if (index > 0) pieces.push(lit(", ")); - pieces.push(...this.printExpr(expr, frame).pieces); - }); - } - - // LIMIT is always emitted and clamped — defense-in-depth (§7 L6). - const limit = Math.min(select.limit ?? MAX_RESULT_ROWS, MAX_RESULT_ROWS); - pieces.push(lit(` LIMIT ${limit}`)); - if (select.offset && select.offset > 0) pieces.push(lit(` OFFSET ${select.offset}`)); - if (select.withTies) { - if (select.orderBy.length === 0) { - throw new VoidQlUnsupportedError({ - message: "LIMIT WITH TIES requires ORDER BY.", - hint: "Add ORDER BY before LIMIT WITH TIES.", - }); - } - pieces.push(lit(" WITH TIES")); - } - - return { pieces, shape, injected: [] }; - } - - private printColumns( - columns: readonly SelectItem[], - frame: Frame, - ): { readonly columnPieces: readonly SqlPiece[]; readonly shape: readonly ColumnSpec[] } { - const columnPieces: SqlPiece[] = []; - const shape: ColumnSpec[] = []; - const outputNames = new Set(); - let emitted = 0; - - const emitSep = () => { - if (emitted > 0) columnPieces.push(lit(", ")); - emitted += 1; - }; - - columns.forEach((item, index) => { - if (item.expr._tag === "StarRef") { - this.expandStar(item.expr.qualifier, frame).forEach((entry) => { - if (outputNames.has(entry.col.toLowerCase())) { - throw new VoidQlUnsupportedError({ - message: `Duplicate output column '${entry.col}'.`, - hint: "Select and alias columns explicitly when relations share names.", - }); - } - outputNames.add(entry.col.toLowerCase()); - emitSep(); - columnPieces.push(lit(this.qualified(entry.alias, entry.col))); - shape.push({ name: entry.col, type: entry.type }); - }); - return; - } - const printed = this.printExpr(item.expr, frame); - emitSep(); - columnPieces.push(...printed.pieces); - const name = item.alias ?? this.inferColumnName(item, index); - // Always emit an explicit `AS ` — including for synthesized names — so - // the ClickHouse output column name equals the reported `shape` name. Without - // it, an unaliased computed column (count(), CASE, arithmetic) is CH-named by - // its expression while `shape` reports `expr_N`, so the caller reads the wrong - // key (silent wrong result) and a subquery/CTE reference to it fails to resolve. - if ( - !IDENT_RE.test(name) || - PHYSICAL_SHAPED_RE.test(name) || - RESERVED_INTERNAL_ALIASES.has(name.toLowerCase()) - ) { - throw new VoidQlUnsupportedError({ message: `Invalid column alias '${name}'.`, hint: "" }); - } - if (outputNames.has(name.toLowerCase())) { - throw new VoidQlUnsupportedError({ - message: `Duplicate output column '${name}'.`, - hint: "Give every selected expression a unique alias.", - }); - } - outputNames.add(name.toLowerCase()); - columnPieces.push(lit(` AS ${name}`)); - shape.push({ name, type: printed.type }); - frame.aliases.set(name.toLowerCase(), printed.type); - }); - - return { columnPieces, shape }; - } - - private inferColumnName(item: SelectItem, index: number): string { - if (item.expr._tag === "ColumnRef") { - const last = item.expr.chain[item.expr.chain.length - 1]!; - return last; - } - return `expr_${index}`; - } - - private expandStar( - qualifier: string | undefined, - frame: Frame, - ): readonly { readonly alias: string; readonly col: string; readonly type: VoidQLType }[] { - let relations: readonly ResolvedRelation[] = frame.relations; - if (qualifier) { - relations = [this.findRelation(frame, qualifier.toLowerCase())].filter( - (r): r is ResolvedRelation => r !== undefined, - ); - } - if (qualifier && relations.length === 0) { - throw new VoidQlUnknownFieldError({ - field: `${qualifier}.*`, - message: `Unknown table '${qualifier}' in '${qualifier}.*'.`, - suggestion: "", - }); - } - return relations.flatMap((rel) => { - if (rel.kind === "view") { - return Object.values(rel.table.columns) - .filter((c) => c.inStar) - .map((c) => ({ alias: rel.alias, col: c.name, type: c.type })); - } - return [...rel.columns.entries()].map(([col, type]) => ({ alias: rel.alias, col, type })); - }); - } -} - -/** - * Resolve + print a parsed statement into the {@link CompiledSelect} IR. Throws the - * typed compile errors (`VoidQlSchemaError`, `VoidQlUnknownFieldError`, - * `VoidQlPiiError`, `VoidQlUnsupportedError`). - */ -export const compileSelect = ( - select: Query, - scope: AuthorizedScope, - capabilities: ReadonlySet, -): CompiledSelect => { - const compiler = new Compiler(scope, capabilities); - const result = compiler.printQuery(select); - return { pieces: result.pieces, shape: result.shape, injected: compiler.injected }; -}; diff --git a/packages/core/src/services/voidql/errors.ts b/packages/core/src/services/voidql/errors.ts deleted file mode 100644 index d8abf9e3f..000000000 --- a/packages/core/src/services/voidql/errors.ts +++ /dev/null @@ -1,154 +0,0 @@ -/** - * VoidQL compile/run domain errors and the structured {@link Diagnostic} the - * agent repair-loop and the editor caret both consume. - * - * Every error is a {@link Schema.TaggedErrorClass} so the RPC layer can translate - * each `_tag` via `Effect.catchTags` into its `Rpc/`-prefixed counterpart (the - * established 3-layer convention). Messages are path-precise but carry **no - * ClickHouse internals** — client-facing surfaces are uniform opaque envelopes to - * starve timing / row-count side-channels (§7 L7, §13). - */ -import { Schema } from "effect"; - -import type { Span } from "./ast/VoidQlAst.ts"; - -/** - * The currency of the agent repair loop and the Monaco red-underline: a typed, - * span-precise compile diagnostic. Returned as *data* (not thrown) by - * `validateQuery`, and mirrored by the thrown error's fields on the run path. - */ -export interface Diagnostic { - /** Which compiler stage produced it. */ - readonly stage: "lex" | "parse" | "resolve" | "verify"; - /** A stable machine code (e.g. `"unknown_field"`, `"pii"`). */ - readonly code: string; - /** Human/agent-readable message. */ - readonly message: string; - /** Source span, when the failure is attributable to one. */ - readonly span?: Span; - /** A teachable hint for self-repair (e.g. "VoidQL is read-only…"). */ - readonly hint?: string; -} - -/** Render a span as a compact `line:col` prefix for flat error messages. */ -export const formatSpan = (span: Span | undefined): string => { - if (span) return `line ${span.start.line}, col ${span.start.col}: `; - return ""; -}; - -/** The query text could not be tokenised or parsed into a valid VoidQL AST. */ -export class VoidQlSyntaxError extends Schema.TaggedErrorClass( - "VoidQlSyntaxError", -)("VoidQlSyntaxError", { message: Schema.String, hint: Schema.String }) {} - -/** - * The query is syntactically reachable but uses a construct VoidQL does not (yet) - * compile — a deferred node, an unknown/denied function, a table function, or a - * `*.table` qualifier. Default-deny: anything not in an allow-list lands here. - */ -export class VoidQlUnsupportedError extends Schema.TaggedErrorClass( - "VoidQlUnsupportedError", -)("VoidQlUnsupportedError", { message: Schema.String, hint: Schema.String }) {} - -/** A relation could not be resolved to a registered logical view. */ -export class VoidQlSchemaError extends Schema.TaggedErrorClass( - "VoidQlSchemaError", -)("VoidQlSchemaError", { message: Schema.String }) {} - -/** A column/property did not resolve; `suggestion` carries the nearest catalog entry. */ -export class VoidQlUnknownFieldError extends Schema.TaggedErrorClass( - "VoidQlUnknownFieldError", -)("VoidQlUnknownFieldError", { - field: Schema.String, - message: Schema.String, - suggestion: Schema.String, -}) {} - -/** - * A PII column/namespace was referenced without the `pii` capability. The *whole* - * query is rejected (never null-substituted) so a `WHERE email = …` cannot act as - * a row-count oracle (§9). - */ -export class VoidQlPiiError extends Schema.TaggedErrorClass("VoidQlPiiError")( - "VoidQlPiiError", - { message: Schema.String }, -) {} - -/** A parser/AST resource cap was exceeded (depth, tokens, nodes, joins, subqueries). */ -export class VoidQlComplexityError extends Schema.TaggedErrorClass( - "VoidQlComplexityError", -)("VoidQlComplexityError", { message: Schema.String }) {} - -/** - * The value-level isolation verifier rejected the compiled statement. This is a - * **compiler defect**, never user error — it must never reach ClickHouse. Surfaced - * to clients as an opaque execution error (§10, §20). - */ -export class VoidQlIsolationError extends Schema.TaggedErrorClass( - "VoidQlIsolationError", -)("VoidQlIsolationError", { message: Schema.String }) {} - -/** Catch-all for execution-time failures (the sanitised ClickHouse-error boundary). */ -export class VoidQlExecutionError extends Schema.TaggedErrorClass( - "VoidQlExecutionError", -)("VoidQlExecutionError", { cause: Schema.String, message: Schema.String }) {} - -/** The union of errors the pure compile pipeline can raise (execution excluded). */ -export type VoidQlCompileError = - | VoidQlSyntaxError - | VoidQlUnsupportedError - | VoidQlSchemaError - | VoidQlUnknownFieldError - | VoidQlPiiError - | VoidQlComplexityError - | VoidQlIsolationError; - -const COMPILE_TAGS = new Set([ - "VoidQlSyntaxError", - "VoidQlUnsupportedError", - "VoidQlSchemaError", - "VoidQlUnknownFieldError", - "VoidQlPiiError", - "VoidQlComplexityError", - "VoidQlIsolationError", -]); - -/** Narrow an unknown thrown value to a VoidQL compile error instance. */ -export const isVoidQlCompileError = (u: unknown): u is VoidQlCompileError => { - if (typeof u !== "object" || u === null) return false; - if (!("_tag" in u)) return false; - const tag = u._tag; - return typeof tag === "string" && COMPILE_TAGS.has(tag); -}; - -/** Renders the "did you mean" hint, or nothing when there is no suggestion. */ -const suggestionHint = (suggestion: string | undefined): string | undefined => { - if (suggestion) return `Did you mean '${suggestion}'?`; - return undefined; -}; - -/** Map a compile error to a public {@link Diagnostic} (used by `validateQuery`). */ -export const toDiagnostic = (error: VoidQlCompileError): Diagnostic => { - switch (error._tag) { - case "VoidQlSyntaxError": - return { stage: "parse", code: "syntax", message: error.message, hint: error.hint }; - case "VoidQlUnsupportedError": - return { stage: "parse", code: "unsupported", message: error.message, hint: error.hint }; - case "VoidQlSchemaError": - return { stage: "resolve", code: "unknown_relation", message: error.message }; - case "VoidQlUnknownFieldError": - return { - stage: "resolve", - code: "unknown_field", - message: error.message, - hint: suggestionHint(error.suggestion), - }; - case "VoidQlPiiError": - return { stage: "resolve", code: "pii", message: error.message }; - case "VoidQlComplexityError": - return { stage: "parse", code: "complexity", message: error.message }; - case "VoidQlIsolationError": - // Never leak the internal reason; the diagnostic is generic. - return { stage: "verify", code: "internal", message: "Query could not be compiled." }; - } -}; diff --git a/packages/core/src/services/voidql/functions.ts b/packages/core/src/services/voidql/functions.ts deleted file mode 100644 index 4aeed415f..000000000 --- a/packages/core/src/services/voidql/functions.ts +++ /dev/null @@ -1,217 +0,0 @@ -/** - * The VoidQL function registry (§11) — a **default-deny** map of VoidQL function - * name → ClickHouse function with arity and result type. A function is callable - * iff it has a row; anything absent (table functions, `dictGet*`, `getSetting`, - * `sleep`, introspection, throwing casts, next year's `azureBlobStorageCluster`) - * is rejected with {@link VoidQlUnsupportedError}. The registry fails *closed* on - * everything new — the inverse of a blocklist, which fails open on every release. - * - * Deferred families (lambdas/higher-order array functions, `arrayJoin`, and - * parametric aggregates) are absent until they land behind a Security-Review Gate. - * Window syntax composes with registry functions; window-only functions still need - * an explicit row here. - */ -import type { VoidQLType } from "./catalog/types.ts"; - -/** A function's result type, or `{ arg: i }` to mean "the type of argument i". */ -export type FnReturn = VoidQLType | { readonly arg: number }; - -export interface FnSpec { - /** The ClickHouse function this lowers to. */ - readonly chName: string; - readonly minArgs: number; - readonly maxArgs: number; - readonly returns: FnReturn; - readonly aggregate: boolean; - /** Whether a bare `*` is an acceptable sole argument (only `count(*)`). */ - readonly allowStar?: boolean; - /** Whether the function is only valid when immediately followed by `OVER`. */ - readonly windowOnly?: boolean; -} - -const REGISTRY: Readonly> = { - // ── aggregations (the core of analytics; pure, deterministic) ── - count: { - chName: "count", - minArgs: 0, - maxArgs: 1, - returns: "UInt64", - aggregate: true, - allowStar: true, - }, - countif: { chName: "countIf", minArgs: 1, maxArgs: 1, returns: "UInt64", aggregate: true }, - countdistinct: { - chName: "uniqExact", - minArgs: 1, - maxArgs: 64, - returns: "UInt64", - aggregate: true, - }, - sum: { chName: "sum", minArgs: 1, maxArgs: 1, returns: "Float64", aggregate: true }, - sumif: { chName: "sumIf", minArgs: 2, maxArgs: 2, returns: "Float64", aggregate: true }, - avg: { chName: "avg", minArgs: 1, maxArgs: 1, returns: "Float64", aggregate: true }, - min: { chName: "min", minArgs: 1, maxArgs: 1, returns: { arg: 0 }, aggregate: true }, - max: { chName: "max", minArgs: 1, maxArgs: 1, returns: { arg: 0 }, aggregate: true }, - any: { chName: "any", minArgs: 1, maxArgs: 1, returns: { arg: 0 }, aggregate: true }, - argmin: { chName: "argMin", minArgs: 2, maxArgs: 2, returns: { arg: 0 }, aggregate: true }, - argmax: { chName: "argMax", minArgs: 2, maxArgs: 2, returns: { arg: 0 }, aggregate: true }, - rownumber: { - chName: "row_number", - minArgs: 0, - maxArgs: 0, - returns: "UInt64", - aggregate: false, - windowOnly: true, - }, - rank: { - chName: "rank", - minArgs: 0, - maxArgs: 0, - returns: "UInt64", - aggregate: false, - windowOnly: true, - }, - denserank: { - chName: "dense_rank", - minArgs: 0, - maxArgs: 0, - returns: "UInt64", - aggregate: false, - windowOnly: true, - }, - - // ── conditional / null handling (pure scalar control flow) ── - if: { chName: "if", minArgs: 3, maxArgs: 3, returns: { arg: 1 }, aggregate: false }, - multiif: { chName: "multiIf", minArgs: 3, maxArgs: 99, returns: { arg: 1 }, aggregate: false }, - coalesce: { chName: "coalesce", minArgs: 1, maxArgs: 99, returns: { arg: 0 }, aggregate: false }, - nullif: { chName: "nullIf", minArgs: 2, maxArgs: 2, returns: { arg: 0 }, aggregate: false }, - ifnull: { chName: "ifNull", minArgs: 2, maxArgs: 2, returns: { arg: 0 }, aggregate: false }, - greatest: { chName: "greatest", minArgs: 2, maxArgs: 99, returns: { arg: 0 }, aggregate: false }, - least: { chName: "least", minArgs: 2, maxArgs: 99, returns: { arg: 0 }, aggregate: false }, - - // ── string / hash (pure) ── - lower: { chName: "lower", minArgs: 1, maxArgs: 1, returns: "String", aggregate: false }, - upper: { chName: "upper", minArgs: 1, maxArgs: 1, returns: "String", aggregate: false }, - length: { chName: "length", minArgs: 1, maxArgs: 1, returns: "UInt64", aggregate: false }, - trim: { chName: "trim", minArgs: 1, maxArgs: 1, returns: "String", aggregate: false }, - concat: { chName: "concat", minArgs: 2, maxArgs: 99, returns: "String", aggregate: false }, - substring: { chName: "substring", minArgs: 2, maxArgs: 3, returns: "String", aggregate: false }, - startswith: { chName: "startsWith", minArgs: 2, maxArgs: 2, returns: "Bool", aggregate: false }, - endswith: { chName: "endsWith", minArgs: 2, maxArgs: 2, returns: "Bool", aggregate: false }, - position: { chName: "position", minArgs: 2, maxArgs: 2, returns: "UInt64", aggregate: false }, - replaceall: { chName: "replaceAll", minArgs: 3, maxArgs: 3, returns: "String", aggregate: false }, - // regex allowed but the query is cost-capped (ReDoS → bounded-time abort, §11). - match: { chName: "match", minArgs: 2, maxArgs: 2, returns: "Bool", aggregate: false }, - replaceregexpall: { - chName: "replaceRegexpAll", - minArgs: 3, - maxArgs: 3, - returns: "String", - aggregate: false, - }, - cityhash64: { - chName: "cityHash64", - minArgs: 1, - maxArgs: 99, - returns: "UInt64", - aggregate: false, - }, - - // ── date / math (pure) ── - tostartofhour: { - chName: "toStartOfHour", - minArgs: 1, - maxArgs: 1, - returns: "DateTime", - aggregate: false, - }, - tostartofminute: { - chName: "toStartOfMinute", - minArgs: 1, - maxArgs: 1, - returns: "DateTime", - aggregate: false, - }, - tostartofday: { - chName: "toStartOfDay", - minArgs: 1, - maxArgs: 1, - returns: "DateTime", - aggregate: false, - }, - tostartofweek: { - chName: "toStartOfWeek", - minArgs: 1, - maxArgs: 2, - returns: "DateTime", - aggregate: false, - }, - tostartofmonth: { - chName: "toStartOfMonth", - minArgs: 1, - maxArgs: 1, - returns: "DateTime", - aggregate: false, - }, - tostartofquarter: { - chName: "toStartOfQuarter", - minArgs: 1, - maxArgs: 1, - returns: "DateTime", - aggregate: false, - }, - tostartofyear: { - chName: "toStartOfYear", - minArgs: 1, - maxArgs: 1, - returns: "DateTime", - aggregate: false, - }, - todate: { chName: "toDate", minArgs: 1, maxArgs: 1, returns: "DateTime", aggregate: false }, - datediff: { chName: "dateDiff", minArgs: 3, maxArgs: 3, returns: "Int64", aggregate: false }, - toyear: { chName: "toYear", minArgs: 1, maxArgs: 1, returns: "UInt64", aggregate: false }, - tomonth: { chName: "toMonth", minArgs: 1, maxArgs: 1, returns: "UInt64", aggregate: false }, - todayofweek: { - chName: "toDayOfWeek", - minArgs: 1, - maxArgs: 1, - returns: "UInt64", - aggregate: false, - }, - round: { chName: "round", minArgs: 1, maxArgs: 2, returns: "Float64", aggregate: false }, - floor: { chName: "floor", minArgs: 1, maxArgs: 2, returns: "Float64", aggregate: false }, - ceil: { chName: "ceil", minArgs: 1, maxArgs: 2, returns: "Float64", aggregate: false }, - abs: { chName: "abs", minArgs: 1, maxArgs: 1, returns: { arg: 0 }, aggregate: false }, - sqrt: { chName: "sqrt", minArgs: 1, maxArgs: 1, returns: "Float64", aggregate: false }, - pow: { chName: "pow", minArgs: 2, maxArgs: 2, returns: "Float64", aggregate: false }, - exp: { chName: "exp", minArgs: 1, maxArgs: 1, returns: "Float64", aggregate: false }, - log: { chName: "log", minArgs: 1, maxArgs: 1, returns: "Float64", aggregate: false }, - // only the null-safe casts (HogQL's leading-underscore rule) — never throwing casts - tofloat64ornull: { - chName: "toFloat64OrNull", - minArgs: 1, - maxArgs: 1, - returns: "Float64", - aggregate: false, - }, - toint64ornull: { - chName: "toInt64OrNull", - minArgs: 1, - maxArgs: 1, - returns: "Int64", - aggregate: false, - }, - todateornull: { - chName: "toDateOrNull", - minArgs: 1, - maxArgs: 1, - returns: "DateTime", - aggregate: false, - }, -}; - -/** Resolve a VoidQL function name (case-insensitive) to its spec, or `undefined`. */ -export const lookupFunction = (name: string): FnSpec | undefined => REGISTRY[name.toLowerCase()]; - -/** All registered VoidQL function names (for agent/editor schema context). */ -export const registeredFunctionNames = (): readonly string[] => Object.keys(REGISTRY); diff --git a/packages/core/src/services/voidql/index.ts b/packages/core/src/services/voidql/index.ts deleted file mode 100644 index 0df8d98f5..000000000 --- a/packages/core/src/services/voidql/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Public entry point for the VoidQL analytics access layer - * (docs/analytics-access-layer.html). A custom, read-only SQL dialect compiled - * server-side to safe ClickHouse SQL, with tenant isolation by logical-view - * substitution (compiler-injected bound-literal scope) + a hardened value-level - * verifier. - */ -export * from "./errors.ts"; -export { compileVoidQl, compilePure, compileToIr, type CompiledQuery } from "./compile.ts"; -export { MAX_RESULT_ROWS } from "./compiler.ts"; -export { CATALOG, CATALOG_SCHEMA_VERSION, getCatalogTable } from "./catalog/index.ts"; -export type { - CatalogTable, - CatalogColumn, - VoidQLType, - Capability, - ColumnSpec, -} from "./catalog/types.ts"; -export { lookupFunction, registeredFunctionNames } from "./functions.ts"; -export { type AuthorizedScope, makeAuthorizedScope } from "./scope.ts"; -export { parse } from "./parser.ts"; -export { lex } from "./lexer.ts"; -export { renderDebugSql, toStatement, type SqlPiece } from "./ir.ts"; -export { verify } from "./verify.ts"; -export { - VoidQlService, - type RunQueryInput, - type RunQueryResult, - type ValidateResult, - type SchemaDescriptor, - type VoidQlPrincipal, -} from "./VoidQlService.ts"; diff --git a/packages/core/src/services/voidql/ir.ts b/packages/core/src/services/voidql/ir.ts deleted file mode 100644 index 7ec3d780d..000000000 --- a/packages/core/src/services/voidql/ir.ts +++ /dev/null @@ -1,75 +0,0 @@ -/** - * The compiled-query intermediate representation: a flat list of {@link SqlPiece}s. - * - * Each piece is either a chunk of compiler-controlled structural SQL (`sql`) or a - * bound parameter (`param`). This *is* the "ResolvedQuery IR" the value-level - * verifier walks (§10, §18 gap #2): it inspects `(sql, binds)` jointly, which a - * placeholder-syntax count cannot. Keeping the printer pure over pieces makes it - * trivially unit-testable and fuzzable; {@link toStatement} is the thin adapter - * that replays the pieces through the audited `ch.literal`/`ch.param` substrate - * for execution, and {@link renderDebugSql} reproduces the exact compiled text + - * binds for the verifier and golden tests. - */ -import type { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import type { Statement } from "effect/unstable/sql/Statement"; - -import { type CatalogSql, catalog } from "./catalog/brand.ts"; - -/** A value bound out-of-band as a ClickHouse named parameter. */ -export type ParamValue = string | number | boolean | readonly string[] | readonly number[]; - -export type SqlPiece = - | { readonly kind: "sql"; readonly text: CatalogSql } - | { readonly kind: "param"; readonly chType: string; readonly value: ParamValue }; - -/** - * Emit structural SQL. The single funnel for `CatalogSql` minting in the printer - * and catalog — only ever fed keywords, validated identifiers, and frozen catalog - * constants, NEVER user text (the §12 invariant; lint-guarded). - */ -export const lit = (text: string): SqlPiece => ({ kind: "sql", text: catalog(text) }); - -/** Emit a bound parameter with an explicit ClickHouse type (§18 gap #9). */ -export const par = (chType: string, value: ParamValue): SqlPiece => ({ - kind: "param", - chType, - value, -}); - -/** - * Render the pieces to the exact compiled SQL text (with `{pN: Type}` placeholders) - * and the ordered bind list — matching what {@link toStatement} + the ClickHouse - * compiler produce. Pure; no `ch` needed. - */ -export const renderDebugSql = ( - pieces: readonly SqlPiece[], -): { readonly sql: string; readonly binds: readonly ParamValue[] } => { - let sql = ""; - const binds: ParamValue[] = []; - for (const piece of pieces) { - if (piece.kind === "sql") { - sql += piece.text; - } else { - binds.push(piece.value); - sql += `{p${binds.length}: ${piece.chType}}`; - } - } - return { sql, binds }; -}; - -/** - * Replay the IR through the audited `ch` substrate, yielding an executable - * {@link Statement}. Structural pieces become `ch.literal(CatalogSql)`; parameters - * become `ch.param(type, value)` — bound out-of-band in `query_params`, never - * spliced. - */ -export const toStatement = >( - ch: ClickhouseWebClient.ClickhouseWebClient, - pieces: readonly SqlPiece[], -): Statement => - pieces.reduce>((frag, piece) => { - if (piece.kind === "sql") { - return ch`${frag}${ch.literal(piece.text)}`; - } - return ch`${frag}${ch.param(piece.chType, piece.value)}`; - }, ch``); diff --git a/packages/core/src/services/voidql/lexer.ts b/packages/core/src/services/voidql/lexer.ts deleted file mode 100644 index 748d5163a..000000000 --- a/packages/core/src/services/voidql/lexer.ts +++ /dev/null @@ -1,326 +0,0 @@ -/** - * The VoidQL lexer — hand-written, `workerd`-pure (no `eval`, no Node deps). - * - * String lexing is **ClickHouse-faithful**: single-quoted, with `''` doubling - * (NOT backslash) escaping a quote. CVE-2025-1520 was a lexer escape mismatch, so - * a borrowed/Postgres lexer is an *isolation* risk, not a correctness bug (§8.1). - * Comments (`-- …`, `/* … *​/`) are dropped here, so the canonical re-print can - * never carry smuggled bytes (T9). - * - * Reserved words — including deferred/denied ones (`SETTINGS`, `ARRAY JOIN`, …) — - * tokenise as keywords so the parser can raise a teachable error rather than - * mis-parsing them as identifiers, and so they can never be used as aliases. - */ -import { pick } from "@voidhash/lib/lang"; -import { Effect } from "effect"; - -import type { Pos } from "./ast/VoidQlAst.ts"; -import { VoidQlComplexityError, VoidQlSyntaxError } from "./errors.ts"; - -/** - * Raises a lexer error out of this synchronous routine. The tagged error is - * surfaced as a defect run synchronously, which rethrows the very instance; - * `compileVoidQl` catches it and re-types it into the compile-error channel. - */ -const raise = (error: VoidQlComplexityError | VoidQlSyntaxError): never => - Effect.runSync(Effect.die(error)); - -export type TokenKind = "ident" | "kw" | "string" | "number" | "op" | "eof"; - -export interface Token { - readonly kind: TokenKind; - /** Keyword tokens are lower-cased; identifiers keep their source case. */ - readonly text: string; - readonly start: Pos; - readonly end: Pos; - /** For `string` tokens: the decoded value. For `number`: the numeric value. */ - readonly value?: string | number; - /** For `number` tokens: the resolved scalar type. */ - readonly numType?: "Int64" | "UInt64" | "Float64"; -} - -/** Structural keywords with a grammar production in the supported query surface. */ -const ALLOWED_KEYWORDS = new Set([ - "select", - "distinct", - "all", - "from", - "prewhere", - "where", - "group", - "by", - "rollup", - "cube", - "totals", - "having", - "qualify", - "order", - "asc", - "desc", - "nulls", - "first", - "last", - "limit", - "offset", - "ties", - "join", - "inner", - "left", - "right", - "full", - "outer", - "cross", - "using", - "on", - "as", - "with", - "partition", - "over", - "rows", - "range", - "unbounded", - "preceding", - "following", - "current", - "row", - "union", - "intersect", - "except", - "and", - "or", - "not", - "in", - "between", - "is", - "null", - "true", - "false", - "case", - "when", - "then", - "else", - "end", - "like", - "ilike", - "exists", -]); - -/** - * Reserved words with **no production** — deferred constructs and permanently - * denied statements. Tokenised so the parser fails closed with a teachable error - * (a parse error, not a parse-then-reject — §8.1, §11, §18 gap #8). - */ -const FORBIDDEN_KEYWORDS = new Set([ - "settings", - "set", - "format", - "into", - "insert", - "update", - "delete", - "alter", - "create", - "drop", - "grant", - "revoke", - "system", - "final", - "sample", - "window", - "array", - "arrayjoin", - "interval", - "values", - "table", - "database", - "attach", - "detach", - "optimize", - "kill", - "show", - "describe", - "explain", - "use", - "outfile", - "infile", -]); - -const KEYWORDS = new Set([...ALLOWED_KEYWORDS, ...FORBIDDEN_KEYWORDS]); - -/** Exposed so the parser can classify a `kw` token without re-deriving the set. */ -export const isForbiddenKeyword = (lower: string): boolean => FORBIDDEN_KEYWORDS.has(lower); - -const MAX_TOKENS = 50_000; - -const isIdentStart = (c: string): boolean => /[A-Za-z_]/.test(c); -const isIdentPart = (c: string): boolean => /[A-Za-z0-9_]/.test(c); -const isDigit = (c: string): boolean => c >= "0" && c <= "9"; - -/** - * Tokenise `text` into a flat token stream terminated by an `eof` token. Throws - * {@link VoidQlSyntaxError} on malformed input and {@link VoidQlComplexityError} - * when the token cap is exceeded (a DoS bound applied *before* parsing). - */ -export const lex = (text: string): readonly Token[] => { - const tokens: Token[] = []; - let offset = 0; - let line = 1; - let col = 1; - - const pos = (): Pos => ({ line, col, offset }); - - const advance = (): string => { - const c = text[offset]!; - offset += 1; - if (c === "\n") { - line += 1; - col = 1; - } else { - col += 1; - } - return c; - }; - - const peek = (ahead = 0): string => text[offset + ahead] ?? ""; - - const push = (token: Token): void => { - if (tokens.length >= MAX_TOKENS) { - return raise(new VoidQlComplexityError({ message: "Query is too large." })); - } - tokens.push(token); - }; - - const fail = (start: Pos, message: string): never => - raise( - new VoidQlSyntaxError({ - message: `line ${start.line}, col ${start.col}: ${message}`, - hint: "", - }), - ); - - while (offset < text.length) { - const c = peek(); - - // whitespace - if (c === " " || c === "\t" || c === "\r" || c === "\n") { - advance(); - continue; - } - - // line comment - if (c === "-" && peek(1) === "-") { - while (offset < text.length && peek() !== "\n") advance(); - continue; - } - - // block comment - if (c === "/" && peek(1) === "*") { - const start = pos(); - advance(); - advance(); - let closed = false; - while (offset < text.length) { - if (peek() === "*" && peek(1) === "/") { - advance(); - advance(); - closed = true; - break; - } - advance(); - } - if (!closed) fail(start, "Unterminated block comment."); - continue; - } - - // string literal — single-quoted, '' escapes a quote, no backslash escaping - if (c === "'") { - const start = pos(); - advance(); - let value = ""; - let closed = false; - while (offset < text.length) { - const ch = advance(); - if (ch === "'") { - if (peek() === "'") { - advance(); - value += "'"; - continue; - } - closed = true; - break; - } - value += ch; - } - if (!closed) fail(start, "Unterminated string literal."); - push({ kind: "string", text: value, value, start, end: pos() }); - continue; - } - - // number literal - if (isDigit(c) || (c === "." && isDigit(peek(1)))) { - const start = pos(); - let raw = ""; - let isFloat = false; - while (offset < text.length && isDigit(peek())) raw += advance(); - if (peek() === ".") { - isFloat = true; - raw += advance(); - while (offset < text.length && isDigit(peek())) raw += advance(); - } - if (peek() === "e" || peek() === "E") { - isFloat = true; - raw += advance(); - if (peek() === "+" || peek() === "-") raw += advance(); - if (!isDigit(peek())) fail(start, "Malformed exponent in number literal."); - while (offset < text.length && isDigit(peek())) raw += advance(); - } - // A trailing identifier char immediately after a number is illegal (e.g. `1abc`). - if (isIdentStart(peek())) fail(start, "Invalid number literal."); - const num = Number(raw); - if (!Number.isFinite(num)) fail(start, "Number literal out of range."); - push({ - kind: "number", - text: raw, - value: num, - numType: pick(isFloat, "Float64", "Int64"), - start, - end: pos(), - }); - continue; - } - - // identifier or keyword - if (isIdentStart(c)) { - const start = pos(); - let raw = ""; - while (offset < text.length && isIdentPart(peek())) raw += advance(); - const lower = raw.toLowerCase(); - if (KEYWORDS.has(lower)) { - push({ kind: "kw", text: lower, start, end: pos() }); - } else { - push({ kind: "ident", text: raw, start, end: pos() }); - } - continue; - } - - // operators / punctuation - const start = pos(); - const two = c + peek(1); - if (two === "<=" || two === ">=" || two === "!=" || two === "<>") { - advance(); - advance(); - push({ kind: "op", text: pick(two === "<>", "!=", two), start, end: pos() }); - continue; - } - if ("=<>+-*/%(),.;".includes(c)) { - advance(); - push({ kind: "op", text: c, start, end: pos() }); - continue; - } - - fail(start, `Unexpected character '${c}'.`); - } - - push({ kind: "eof", text: "", start: pos(), end: pos() }); - return tokens; -}; diff --git a/packages/core/src/services/voidql/parser.ts b/packages/core/src/services/voidql/parser.ts deleted file mode 100644 index ce67dada7..000000000 --- a/packages/core/src/services/voidql/parser.ts +++ /dev/null @@ -1,888 +0,0 @@ -/* - * Recursive-descent parsers use exceptions as their non-local exit: a bad token - * anywhere in the descent must unwind straight out of the nested `parse*` - * frames. Throwing is therefore the control flow of this entire module, not an - * escape hatch at a few sites. `compile.ts` is the single Effect boundary that - * converts these tagged VoidQL errors back into typed failures; routing them - * through Effect here would surface every compile error as an opaque defect and - * force each of the ~60 mutually recursive methods to become an Effect. - */ -// oxlint-disable effect/noThrowStatement -- throw IS the parser's control flow (see block comment above); compile.ts is the single Effect boundary that converts it. -/** - * The VoidQL parser — hand-written recursive descent with a Pratt - * (precedence-climbing) expression core. No `eval`, no Node deps; `workerd`-pure. - * - * This module is the **sole, trusted constructor of AST nodes** (§8.2): no other - * module is permitted to build a node literal, which is what makes "user text - * never reaches `ch.literal`" a static property rather than an assumption. - * - * Depth / node / join / subquery caps live **in the cursor**, checked *before* - * recursion, so a 10⁵-deep input is a typed {@link VoidQlComplexityError}, not a - * JS-stack blow-up (§18 gap #6). Deferred and denied constructs (`SETTINGS`, - * `ARRAY JOIN`, named windows, …) have no production at all — they are parse - * errors, never parse-then-reject, so the highest-risk surface never has a - * validator to bypass. - */ -import { constant, numberOr, stringOr } from "@voidhash/lib/lang"; - -import type { - Binary, - BinaryOp, - CaseWhen, - Cte, - Expr, - FnCall, - Join, - OrderItem, - Pos, - Query, - Select, - SelectItem, - SetOperator, - Span, - Statement, - TableSource, - WindowFrame, - WindowFrameBound, -} from "./ast/VoidQlAst.ts"; -import { VoidQlComplexityError, VoidQlSyntaxError, VoidQlUnsupportedError } from "./errors.ts"; -import { isForbiddenKeyword, lex, type Token } from "./lexer.ts"; - -const LIMITS = constant({ - maxDepth: 50, - maxNodes: 20_000, - maxJoins: 8, - maxSubqueries: 16, -}); - -class Cursor { - private index = 0; - private depth = 0; - private nodeCount = 0; - private joinCount = 0; - private subqueryCount = 0; - private readonly tokens: readonly Token[]; - - constructor(tokens: readonly Token[]) { - this.tokens = tokens; - } - - // ── token access ────────────────────────────────────────────────────────── - - private peek(ahead = 0): Token { - return this.tokens[Math.min(this.index + ahead, this.tokens.length - 1)]!; - } - - private prev(): Token { - return this.tokens[Math.max(this.index - 1, 0)]!; - } - - private next(): Token { - const token = this.peek(); - if (token.kind !== "eof") this.index += 1; - return token; - } - - private isOp(text: string): boolean { - const t = this.peek(); - return t.kind === "op" && t.text === text; - } - - private isKw(text: string): boolean { - const t = this.peek(); - return t.kind === "kw" && t.text === text; - } - - private eatOp(text: string): boolean { - if (this.isOp(text)) { - this.index += 1; - return true; - } - return false; - } - - private eatKw(text: string): boolean { - if (this.isKw(text)) { - this.index += 1; - return true; - } - return false; - } - - private fail(token: Token, message: string, hint = ""): never { - throw new VoidQlSyntaxError({ - message: `line ${token.start.line}, col ${token.start.col}: ${message}`, - hint, - }); - } - - private expectOp(text: string): void { - if (!this.eatOp(text)) this.fail(this.peek(), `Expected '${text}'.`); - } - - private expectKw(text: string): void { - if (!this.eatKw(text)) this.fail(this.peek(), `Expected '${text.toUpperCase()}'.`); - } - - private expectIdent(what: string): string { - const t = this.peek(); - if (t.kind !== "ident") this.fail(t, `Expected ${what}.`); - this.index += 1; - return t.text; - } - - private span(start: Pos): Span { - return { start, end: this.prev().end }; - } - - /** - * Increment the node counter and fail closed past the cap. Every node literal - * is stamped through here, so the bound is enforced at construction. - */ - private count(): void { - this.nodeCount += 1; - if (this.nodeCount > LIMITS.maxNodes) { - throw new VoidQlComplexityError({ message: "Query has too many elements." }); - } - } - - private enter(): void { - this.depth += 1; - if (this.depth > LIMITS.maxDepth) { - throw new VoidQlComplexityError({ message: "Query is nested too deeply." }); - } - } - - private leave(): void { - this.depth -= 1; - } - - /** - * Reject deferred/denied keywords with a teachable message before they can be - * mis-parsed as the start of an expression. - */ - private forbiddenHint(text: string): string { - if (text === "settings" || text === "set") { - return "VoidQL does not support a SETTINGS clause; tenant scope is applied automatically and cannot be set in a query."; - } - return `VoidQL is a read-only subset; '${text.toUpperCase()}' is not part of the dialect.`; - } - - /** - * Runs `production` when `keyword` is the next token, otherwise yields - * `undefined` — the optional-clause shape shared by FROM/WHERE/HAVING/…. - */ - private after(keyword: string, production: () => A): A | undefined { - if (this.eatKw(keyword)) { - return production(); - } - return undefined; - } - - private guardForbidden(): void { - const t = this.peek(); - if (t.kind === "kw" && isForbiddenKeyword(t.text)) { - const hint = this.forbiddenHint(t.text); - throw new VoidQlUnsupportedError({ - message: `line ${t.start.line}, col ${t.start.col}: Unsupported keyword '${t.text.toUpperCase()}'.`, - hint, - }); - } - } - - // ── statement ────────────────────────────────────────────────────────────── - - parseStatement(): Statement { - const query = this.parseQuery(); - this.eatOp(";"); - const tail = this.peek(); - if (tail.kind !== "eof") { - // A trailing token after a complete SELECT is almost always a denied clause - // (e.g. `… SETTINGS x=1` or `… FORMAT JSON`). - this.guardForbidden(); - this.fail(tail, `Unexpected '${tail.text || tail.kind}' after end of query.`); - } - return query; - } - - private parseSetOperator(): SetOperator { - if (this.eatKw("union")) { - if (this.eatKw("all")) return "UNION ALL"; - if (this.eatKw("distinct")) return "UNION DISTINCT"; - throw new VoidQlUnsupportedError({ - message: `line ${this.peek().start.line}, col ${this.peek().start.col}: UNION requires ALL or DISTINCT.`, - hint: "Specify UNION ALL or UNION DISTINCT explicitly.", - }); - } - if (this.eatKw("intersect")) { - this.eatKw("distinct"); - return "INTERSECT"; - } - this.expectKw("except"); - this.eatKw("distinct"); - return "EXCEPT"; - } - - private parseQuery(): Query { - const first = this.parseSelect(); - // Operator/select pairs, so the non-empty tuple shapes SetQuery requires fall - // out of the parse instead of needing an assertion. - const tail: Array<{ readonly operator: SetOperator; readonly select: Select }> = []; - while (this.isKw("union") || this.isKw("intersect") || this.isKw("except")) { - const operator = this.parseSetOperator(); - tail.push({ operator, select: this.parseSelect() }); - } - const [head, ...rest] = tail; - if (head === undefined) return first; - this.count(); - return { - _tag: "SetQuery", - selects: [first, head.select, ...rest.map((entry) => entry.select)], - operators: [head.operator, ...rest.map((entry) => entry.operator)], - span: { start: first.span.start, end: tail[tail.length - 1]!.select.span.end }, - }; - } - - private parseSelect(): Select { - this.enter(); - const start = this.peek().start; - - const ctes: Cte[] = []; - if (this.eatKw("with")) { - do { - const cteStart = this.peek().start; - const name = this.expectIdent("a CTE name"); - this.expectKw("as"); - this.expectOp("("); - const query = this.parseQuery(); - this.expectOp(")"); - this.count(); - ctes.push({ _tag: "Cte", name, query, span: this.span(cteStart) }); - } while (this.eatOp(",")); - } - - this.guardForbidden(); // a leading denied keyword (DROP/INSERT/…) → teachable error - this.expectKw("select"); - const distinct = this.eatKw("distinct"); - const distinctOn: Expr[] = []; - if (distinct && this.eatKw("on")) { - this.expectOp("("); - distinctOn.push(this.parseExpr(0)); - while (this.eatOp(",")) distinctOn.push(this.parseExpr(0)); - this.expectOp(")"); - } else if (!distinct) { - this.eatKw("all"); - } - this.guardForbidden(); - - const columns: [SelectItem, ...SelectItem[]] = [this.parseSelectItem()]; - while (this.eatOp(",")) columns.push(this.parseSelectItem()); - - const from = this.after("from", () => this.parseTableSource()); - - const joins: Join[] = []; - if (from) { - for (;;) { - const join = this.tryParseJoin(); - if (!join) break; - if (joins.length >= LIMITS.maxJoins) { - throw new VoidQlComplexityError({ message: "Query has too many joins." }); - } - joins.push(join); - } - } - - const prewhere = this.after("prewhere", () => this.parseExpr(0)); - const where = this.after("where", () => this.parseExpr(0)); - - const groupBy: Expr[] = []; - if (this.eatKw("group")) { - this.expectKw("by"); - groupBy.push(this.parseExpr(0)); - while (this.eatOp(",")) groupBy.push(this.parseExpr(0)); - } - let groupByModifier: Select["groupByModifier"]; - let withTotals = false; - if (groupBy.length > 0) { - while (this.eatKw("with")) { - if (this.eatKw("rollup")) groupByModifier = "rollup"; - else if (this.eatKw("cube")) groupByModifier = "cube"; - else if (this.eatKw("totals")) withTotals = true; - else this.fail(this.peek(), "Expected ROLLUP, CUBE, or TOTALS after WITH."); - } - } - - const having = this.after("having", () => this.parseExpr(0)); - const qualify = this.after("qualify", () => this.parseExpr(0)); - - const orderBy: OrderItem[] = []; - if (this.eatKw("order")) { - this.expectKw("by"); - orderBy.push(this.parseOrderItem()); - while (this.eatOp(",")) orderBy.push(this.parseOrderItem()); - } - - let limitBy: Select["limitBy"]; - let limit: number | undefined; - let offset: number | undefined; - let withTies = false; - if (this.eatKw("limit")) { - const first = this.parseUintLiteral("a LIMIT count"); - let second: number | undefined; - if (this.eatOp(",")) second = this.parseUintLiteral("a LIMIT count after the offset"); - if (this.eatKw("by")) { - const by: [Expr, ...Expr[]] = [this.parseExpr(0)]; - while (this.eatOp(",")) by.push(this.parseExpr(0)); - // `LIMIT n, m BY …` puts the offset first; `LIMIT n BY …` has none. - let byOffset: number | undefined; - if (second !== undefined) byOffset = first; - limitBy = { - limit: second ?? first, - offset: byOffset, - by, - }; - if (this.eatKw("limit")) { - const finalFirst = this.parseUintLiteral("a LIMIT count"); - if (this.eatOp(",")) { - offset = finalFirst; - limit = this.parseUintLiteral("a LIMIT count after the offset"); - } else { - limit = finalFirst; - if (this.eatKw("offset")) offset = this.parseUintLiteral("an OFFSET count"); - } - } - } else if (second !== undefined) { - offset = first; - limit = second; - } else { - limit = first; - if (this.eatKw("offset")) offset = this.parseUintLiteral("an OFFSET count"); - } - } - if (limit !== undefined && this.eatKw("with")) { - this.expectKw("ties"); - withTies = true; - } - - this.count(); - this.leave(); - return { - _tag: "Select", - with: ctes, - distinct, - distinctOn, - columns, - from, - joins, - prewhere, - where, - groupBy, - groupByModifier, - withTotals, - having, - qualify, - orderBy, - limitBy, - limit, - offset, - withTies, - span: this.span(start), - }; - } - - private parseUintLiteral(what: string): number { - const t = this.peek(); - const value = t.value; - if (t.kind !== "number" || t.numType !== "Int64" || typeof value !== "number" || value < 0) { - this.fail(t, `Expected ${what} (a non-negative integer).`); - } - this.index += 1; - return value; - } - - private parseSelectItem(): SelectItem { - const start = this.peek().start; - const expr = this.parseExpr(0); - let alias: string | undefined; - if (this.eatKw("as")) { - alias = this.expectIdent("a column alias"); - } else if (this.peek().kind === "ident") { - alias = this.next().text; - } - this.count(); - return { _tag: "SelectItem", expr, alias, span: this.span(start) }; - } - - private parseOrderItem(): OrderItem { - const start = this.peek().start; - const expr = this.parseExpr(0); - let dir: "asc" | "desc" = "asc"; - if (this.eatKw("asc")) dir = "asc"; - else if (this.eatKw("desc")) dir = "desc"; - let nulls: OrderItem["nulls"]; - if (this.eatKw("nulls")) { - if (this.eatKw("first")) nulls = "first"; - else if (this.eatKw("last")) nulls = "last"; - else this.fail(this.peek(), "Expected FIRST or LAST after NULLS."); - } - this.count(); - return { _tag: "OrderItem", expr, dir, nulls, span: this.span(start) }; - } - - private tryParseJoin(): Join | undefined { - let kind: Join["kind"]; - if (this.isKw("inner")) { - this.next(); - kind = "inner"; - this.expectKw("join"); - } else if (this.isKw("left")) { - this.next(); - kind = "left"; - this.eatKw("outer"); - this.expectKw("join"); - } else if (this.isKw("right")) { - this.next(); - kind = "right"; - this.eatKw("outer"); - this.expectKw("join"); - } else if (this.isKw("full")) { - this.next(); - kind = "full"; - this.eatKw("outer"); - this.expectKw("join"); - } else if (this.isKw("cross")) { - this.next(); - kind = "cross"; - this.expectKw("join"); - } else if (this.isKw("join")) { - this.next(); - kind = "inner"; - } else { - return undefined; - } - const start = this.prev().start; - const source = this.parseTableSource(); - let on: Expr | undefined; - let using: [string, ...string[]] | undefined; - if (kind !== "cross") { - if (this.eatKw("on")) { - on = this.parseExpr(0); - } else if (this.eatKw("using")) { - const parenthesized = this.eatOp("("); - using = [this.expectIdent("a column name in USING")]; - while (this.eatOp(",")) using.push(this.expectIdent("a column name in USING")); - if (parenthesized) this.expectOp(")"); - } else { - this.fail(this.peek(), "Expected 'ON' or 'USING' after JOIN source."); - } - } - this.count(); - return { - _tag: "Join", - kind, - source, - on, - using, - span: this.span(start), - }; - } - - private parseTableSource(): TableSource { - const start = this.peek().start; - if (this.eatOp("(")) { - if (++this.subqueryCount > LIMITS.maxSubqueries) { - throw new VoidQlComplexityError({ message: "Query has too many subqueries." }); - } - if (!this.isKw("select") && !this.isKw("with")) { - this.fail(this.peek(), "Expected a subquery (SELECT …) after '('."); - } - const query = this.parseQuery(); - this.expectOp(")"); - this.eatKw("as"); - const alias = this.expectIdent("an alias for the subquery (subqueries must be aliased)"); - this.count(); - return { _tag: "SubquerySource", query, alias, span: this.span(start) }; - } - this.guardForbidden(); // a forbidden keyword in FROM position (e.g. a table fn name shaped like a kw) - const name = this.expectIdent("a table name"); - let alias: string | undefined; - if (this.eatKw("as")) alias = this.expectIdent("a table alias"); - else if (this.peek().kind === "ident") alias = this.next().text; - this.count(); - return { _tag: "NamedTable", name, alias, span: this.span(start) }; - } - - // ── expressions (Pratt) ────────────────────────────────────────────────── - - parseExpr(minPrec: number): Expr { - this.enter(); - let left = this.parsePrefix(); - for (;;) { - const next = this.parseInfix(left, minPrec); - if (next === undefined) break; - left = next; - } - this.leave(); - return left; - } - - private mkBinary(op: BinaryOp, left: Expr, right: Expr, start: Pos): Binary { - this.count(); - return { _tag: "Binary", op, left, right, span: this.span(start) }; - } - - /** One infix step; returns `undefined` to stop the precedence loop. */ - private parseInfix(left: Expr, minPrec: number): Expr | undefined { - const t = this.peek(); - const start = left.span.start; - - // keyword infix operators sit at comparison precedence (3) - if (3 > minPrec && t.kind === "kw") { - if (t.text === "in") { - this.next(); - return this.parseInTail(left, false, start); - } - if (t.text === "between") { - this.next(); - return this.parseBetweenTail(left, false, start); - } - if (t.text === "like") { - this.next(); - return this.mkBinary("like", left, this.parseExpr(3), start); - } - if (t.text === "ilike") { - this.next(); - return this.mkBinary("ilike", left, this.parseExpr(3), start); - } - if (t.text === "is") { - this.next(); - const negated = this.eatKw("not"); - this.expectKw("null"); - this.count(); - return { _tag: "IsNull", expr: left, negated, span: this.span(start) }; - } - if (t.text === "not" && this.peek(1).kind === "kw") { - const follow = this.peek(1).text; - if (follow === "in") { - this.next(); - this.next(); - return this.parseInTail(left, true, start); - } - if (follow === "between") { - this.next(); - this.next(); - return this.parseBetweenTail(left, true, start); - } - if (follow === "like") { - this.next(); - this.next(); - return this.mkBinary("notLike", left, this.parseExpr(3), start); - } - if (follow === "ilike") { - this.next(); - this.next(); - return this.mkBinary("notIlike", left, this.parseExpr(3), start); - } - } - } - - if (t.kind === "kw" && t.text === "or" && 1 > minPrec) { - this.next(); - return this.mkBinary("or", left, this.parseExpr(1), start); - } - if (t.kind === "kw" && t.text === "and" && 2 > minPrec) { - this.next(); - return this.mkBinary("and", left, this.parseExpr(2), start); - } - - if (t.kind === "op") { - const cmp: Record = { - "=": "eq", - "!=": "neq", - "<": "lt", - "<=": "lte", - ">": "gt", - ">=": "gte", - }; - if (cmp[t.text] && 3 > minPrec) { - this.next(); - return this.mkBinary(cmp[t.text]!, left, this.parseExpr(3), start); - } - if (t.text === "+" && 4 > minPrec) { - this.next(); - return this.mkBinary("add", left, this.parseExpr(4), start); - } - if (t.text === "-" && 4 > minPrec) { - this.next(); - return this.mkBinary("sub", left, this.parseExpr(4), start); - } - if (t.text === "*" && 5 > minPrec) { - this.next(); - return this.mkBinary("mul", left, this.parseExpr(5), start); - } - if (t.text === "/" && 5 > minPrec) { - this.next(); - return this.mkBinary("div", left, this.parseExpr(5), start); - } - if (t.text === "%" && 5 > minPrec) { - this.next(); - return this.mkBinary("mod", left, this.parseExpr(5), start); - } - } - - return undefined; - } - - private parseInTail(left: Expr, negated: boolean, start: Pos): Expr { - this.expectOp("("); - if (this.isKw("select") || this.isKw("with")) { - const query = this.parseQuery(); - this.expectOp(")"); - this.count(); - return { _tag: "InExpr", expr: left, query, negated, span: this.span(start) }; - } - const list: [Expr, ...Expr[]] = [this.parseExpr(0)]; - while (this.eatOp(",")) list.push(this.parseExpr(0)); - this.expectOp(")"); - this.count(); - return { - _tag: "InExpr", - expr: left, - list, - negated, - span: this.span(start), - }; - } - - private parseBetweenTail(left: Expr, negated: boolean, start: Pos): Expr { - const low = this.parseExpr(3); - this.expectKw("and"); - const high = this.parseExpr(3); - this.count(); - return { _tag: "Between", expr: left, low, high, negated, span: this.span(start) }; - } - - private parsePrefix(): Expr { - this.guardForbidden(); - const t = this.peek(); - const start = t.start; - - if (t.kind === "kw") { - switch (t.text) { - case "not": - this.next(); - this.count(); - return { _tag: "Unary", op: "not", expr: this.parseExpr(2), span: this.span(start) }; - case "true": - case "false": - this.next(); - this.count(); - return { _tag: "BoolLit", value: t.text === "true", span: this.span(start) }; - case "null": - this.next(); - this.count(); - return { _tag: "NullLit", span: this.span(start) }; - case "case": - return this.parseCase(); - case "exists": { - this.next(); - this.expectOp("("); - if (!this.isKw("select") && !this.isKw("with")) { - this.fail(this.peek(), "EXISTS requires a SELECT subquery."); - } - const query = this.parseQuery(); - this.expectOp(")"); - this.count(); - return { _tag: "ExistsExpr", query, span: this.span(start) }; - } - default: - this.fail(t, `Unexpected keyword '${t.text.toUpperCase()}'.`); - } - } - - if (t.kind === "op") { - if (t.text === "-") { - this.next(); - this.count(); - // Depth-guard the prefix recursion: unlike `not` (which recurses through - // `parseExpr`), `neg` recurses `parsePrefix` directly, so without enter()/ - // leave() a long `- - … - x` chain is bounded only by maxNodes (20k) and - // blows the JS stack with a raw RangeError before that cap fires. - this.enter(); - const operand = this.parsePrefix(); - this.leave(); - return { _tag: "Unary", op: "neg", expr: operand, span: this.span(start) }; - } - if (t.text === "(") { - this.next(); - if (this.isKw("select") || this.isKw("with")) { - const query = this.parseQuery(); - this.expectOp(")"); - this.count(); - return { _tag: "SubqueryExpr", query, span: this.span(start) }; - } - const expr = this.parseExpr(0); - this.expectOp(")"); - this.count(); - return { _tag: "Paren", expr, span: this.span(start) }; - } - if (t.text === "*") { - this.next(); - this.count(); - return { _tag: "StarRef", span: this.span(start) }; - } - this.fail(t, `Unexpected '${t.text}'.`); - } - - if (t.kind === "string") { - this.next(); - this.count(); - return { _tag: "StringLit", value: stringOr(t.value, ""), span: this.span(start) }; - } - - if (t.kind === "number") { - this.next(); - this.count(); - return { - _tag: "NumberLit", - value: numberOr(t.value, 0), - numType: t.numType ?? "Int64", - span: this.span(start), - }; - } - - if (t.kind === "ident") { - // function call vs column reference - if (this.peek(1).kind === "op" && this.peek(1).text === "(") { - return this.parseFnCall(); - } - return this.parseColumnRef(); - } - - this.fail(t, "Expected an expression."); - } - - private parseFnCall(): Expr { - const start = this.peek().start; - const name = this.next().text; - this.expectOp("("); - const args: Expr[] = []; - if (!this.isOp(")")) { - args.push(this.parseExpr(0)); - while (this.eatOp(",")) args.push(this.parseExpr(0)); - } - this.expectOp(")"); - this.count(); - const fn: FnCall = { _tag: "FnCall", name, args, span: this.span(start) }; - if (!this.eatKw("over")) return fn; - - this.expectOp("("); - const partitionBy: Expr[] = []; - if (this.eatKw("partition")) { - this.expectKw("by"); - partitionBy.push(this.parseExpr(0)); - while (this.eatOp(",")) partitionBy.push(this.parseExpr(0)); - } - const orderBy: OrderItem[] = []; - if (this.eatKw("order")) { - this.expectKw("by"); - orderBy.push(this.parseOrderItem()); - while (this.eatOp(",")) orderBy.push(this.parseOrderItem()); - } - let frame: WindowFrame | undefined; - const unit = this.tryParseWindowUnit(); - if (unit !== undefined) { - if (this.eatKw("between")) { - const frameStart = this.parseWindowFrameBound(); - this.expectKw("and"); - frame = { unit, start: frameStart, end: this.parseWindowFrameBound() }; - } else { - frame = { unit, start: this.parseWindowFrameBound() }; - } - } - this.expectOp(")"); - this.count(); - return { - _tag: "WindowExpr", - fn, - partitionBy, - orderBy, - frame, - span: this.span(start), - }; - } - - private tryParseWindowUnit(): WindowFrame["unit"] | undefined { - if (this.eatKw("rows")) return "rows"; - if (this.eatKw("range")) return "range"; - return undefined; - } - - private parseWindowFrameBound(): WindowFrameBound { - if (this.eatKw("current")) { - this.expectKw("row"); - return "currentRow"; - } - if (this.eatKw("unbounded")) { - if (this.eatKw("preceding")) return "unboundedPreceding"; - if (this.eatKw("following")) return "unboundedFollowing"; - } - throw new VoidQlUnsupportedError({ - message: `line ${this.peek().start.line}, col ${this.peek().start.col}: Unsupported window frame bound.`, - hint: "Use UNBOUNDED PRECEDING, CURRENT ROW, or UNBOUNDED FOLLOWING.", - }); - } - - private parseColumnRef(): Expr { - const start = this.peek().start; - const chain: [string, ...string[]] = [this.next().text]; - while (this.isOp(".")) { - this.next(); - if (this.eatOp("*")) { - this.count(); - return { _tag: "StarRef", qualifier: chain.join("."), span: this.span(start) }; - } - const seg = this.peek(); - if (seg.kind !== "ident") this.fail(seg, "Expected a column or property name after '.'."); - this.index += 1; - chain.push(seg.text); - } - this.count(); - return { _tag: "ColumnRef", chain, span: this.span(start) }; - } - - private parseCaseWhenBranch(): CaseWhen { - const when = this.parseExpr(0); - this.expectKw("then"); - const then = this.parseExpr(0); - // oxlint-disable-next-line unicorn/no-thenable -- `then` is the SQL CASE ... WHEN ... THEN branch of the frozen `CaseWhen` AST node; renaming the field would change the AST contract every consumer and the compiler match on. - return { when, then }; - } - - private parseCase(): Expr { - const start = this.peek().start; - this.expectKw("case"); - let operand: Expr | undefined; - if (!this.isKw("when")) operand = this.parseExpr(0); - if (!this.eatKw("when")) this.fail(this.peek(), "CASE requires at least one WHEN branch."); - const whens: [CaseWhen, ...CaseWhen[]] = [this.parseCaseWhenBranch()]; - while (this.eatKw("when")) whens.push(this.parseCaseWhenBranch()); - const elseExpr = this.after("else", () => this.parseExpr(0)); - this.expectKw("end"); - this.count(); - return { - _tag: "CaseExpr", - operand, - whens, - else: elseExpr, - span: this.span(start), - }; - } -} - -/** - * Parse VoidQL query text into a {@link Statement}. Throws - * {@link VoidQlSyntaxError}, {@link VoidQlUnsupportedError}, or - * {@link VoidQlComplexityError} as typed values (never a string). - */ -export const parse = (text: string): Statement => new Cursor(lex(text)).parseStatement(); diff --git a/packages/core/src/services/voidql/scope.ts b/packages/core/src/services/voidql/scope.ts deleted file mode 100644 index 07e5a0765..000000000 --- a/packages/core/src/services/voidql/scope.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** - * {@link AuthorizedScope} — the single, server-derived source for the injected - * `organization_id = {pOrg}` / `project_id IN {pPids}` bound literals (§10, §18 - * gap #12). It is *nominally branded* so it can only be minted by {@link makeAuthorizedScope}, - * which the service calls **after** `checkOrganizationPermission` succeeds — there - * is no second tenant-setting mechanism to keep in sync, so the substitution is - * derived from one value the user cannot influence. - */ -import { Brand } from "effect"; - -/** The structural payload an {@link AuthorizedScope} carries. */ -export interface AuthorizedScopeFields { - /** The single authorized organization id. */ - readonly organizationId: string; - /** Exactly the projects the caller may read — the `project_id IN (…)` allow-set. */ - readonly availableProjectIds: readonly string[]; -} - -export type AuthorizedScope = Brand.Branded; - -const authorizedScope = Brand.nominal(); - -/** - * Construct an {@link AuthorizedScope}. MUST only be called after an affirmative - * `checkOrganizationPermission` against the *authorized* org (never the request - * body) — enforced by convention in `VoidQlService.buildScope`. - */ -export const makeAuthorizedScope = (input: AuthorizedScopeFields): AuthorizedScope => - authorizedScope({ - organizationId: input.organizationId, - availableProjectIds: input.availableProjectIds, - }); diff --git a/packages/core/src/services/voidql/verify.ts b/packages/core/src/services/voidql/verify.ts deleted file mode 100644 index eefae7a4f..000000000 --- a/packages/core/src/services/voidql/verify.ts +++ /dev/null @@ -1,114 +0,0 @@ -/** - * The value-level isolation verifier (§10) — the **sole in-process fail-closed - * net** now that the out-of-process row policy is gone (§20). It walks the - * compiled {@link SqlPiece} IR, rendering `(sql, binds)` jointly, and hard-fails - * the request (a {@link VoidQlIsolationError} defect that never reaches ClickHouse) - * unless every invariant holds. - * - * Crucially it does NOT trust the printer's self-reported scopes alone: it - * **triangulates** the printer-reported base-ref count against the physical-table - * tokens it independently finds in the emitted SQL *and* the tenant-predicate - * occurrences in that SQL. A printer bug that emits a base table without injecting - * a scope therefore fails closed here rather than leaking. - * - * Because this is the only net, its per-occurrence enumeration must grow in - * lock-step with every newly enabled construct (UNION arms, lambdas, windows) — - * a blind spot is a direct leak, not a caught one (§11 G1, §20). - */ -import { Effect } from "effect"; - -import type { InjectedScope } from "./catalog/types.ts"; -import { VoidQlIsolationError } from "./errors.ts"; -import { renderDebugSql, type SqlPiece } from "./ir.ts"; -import type { AuthorizedScope } from "./scope.ts"; - -/** Physical tables VoidQL is permitted to reference — all only inside a `lower()`. */ -const ALLOWED_PHYSICAL_TABLES = new Set([ - "events_v2", - "persons_v1", - "person_identity_pending_overrides_v2", -]); - -/** Tokens that must never appear in compiled VoidQL — the catalog/registry already - * exclude them; this scan is the independent backstop (T3/T4/T5/T8). */ -const FORBIDDEN_TOKEN_RE = - /\b(remote|remoteSecure|url|s3|s3Cluster|file|mysql|postgresql|jdbc|odbc|hdfs|azureBlobStorage|mongodb|cluster|clusterAllReplicas|numbers|generateRandom|dictGet\w*|dictHas|getSetting|currentUser|hostName|serverUUID|getMacro|addressToLine|demangle|evalMLMethod|sleep|sleepEachRow|throwIf)\s*\(/i; - -const VERSIONED_TABLE_RE = /\b\w+_v\d+\b/g; - -const countOccurrences = (haystack: string, needle: string): number => - haystack.split(needle).length - 1; - -const arrayEqual = (a: readonly string[], b: readonly string[]): boolean => - a.length === b.length && a.every((v, i) => v === b[i]); - -const fail = (message: string): never => - // `Effect.runSync` on a died effect rethrows the defect verbatim, so callers - // still observe the tagged `VoidQlIsolationError` itself. - Effect.runSync(Effect.die(new VoidQlIsolationError({ message: `isolation verifier: ${message}` }))); - -/** - * Verify the compiled IR against the authorized scope. Throws - * {@link VoidQlIsolationError} (a compiler defect) on any violation. - */ -export const verify = ( - pieces: readonly SqlPiece[], - injected: readonly InjectedScope[], - scope: AuthorizedScope, -): void => { - const { sql } = renderDebugSql(pieces); - - // ── I1: every injected scope binds to the authorized org and exactly its projects. - for (const scoped of injected) { - if (scoped.orgValue !== scope.organizationId) { - fail(`base-ref '${scoped.alias}' bound to a non-authorized organization`); - } - if (!arrayEqual(scoped.projectValues, scope.availableProjectIds)) { - fail(`base-ref '${scoped.alias}' bound to the wrong project set`); - } - } - - // ── Triangulation: printer-reported base-refs vs physical tables vs predicates. - // An `events`/`revenue` lowering scopes TWO physical reads — the dedup scan of - // `events_v2` AND the identity-join scan of `person_identity_pending_overrides_v2` - // (the latter has no row policy under the analytics_query user, so it is scoped - // inline, §9/§20). A `persons` lowering scopes ONE (`persons_v1`). So each - // event-backed scope contributes 2 tenant predicates and 1 of each physical - // table; each person-backed scope contributes 1 predicate and 1 `persons_v1`. - const eventBacked = injected.filter( - (s) => s.relation === "events" || s.relation === "revenue", - ).length; - const personBacked = injected.filter((s) => s.relation === "persons").length; - const expectedPredicates = eventBacked * 2 + personBacked; - - if (countOccurrences(sql, "events_v2") !== eventBacked) { - fail("events_v2 occurrence count does not match injected event scopes"); - } - if (countOccurrences(sql, "person_identity_pending_overrides_v2") !== eventBacked) { - fail("pending-overrides occurrence count does not match injected event scopes"); - } - if (countOccurrences(sql, "persons_v1") !== personBacked) { - fail("persons_v1 occurrence count does not match injected person scopes"); - } - if (countOccurrences(sql, "organization_id = {p") !== expectedPredicates) { - fail("organization_id predicate count does not match scoped physical-read count"); - } - if (countOccurrences(sql, "project_id IN {p") !== expectedPredicates) { - fail("project_id predicate count does not match scoped physical-read count"); - } - - // ── I2: only the allow-listed physical tables appear; no table fns / introspection. - for (const match of sql.matchAll(VERSIONED_TABLE_RE)) { - if (!ALLOWED_PHYSICAL_TABLES.has(match[0])) { - fail(`unexpected physical table token '${match[0]}'`); - } - } - if (/\bsystem\./i.test(sql)) fail("reference to a system table"); - const forbidden = FORBIDDEN_TOKEN_RE.exec(sql); - if (forbidden) fail(`forbidden function/table-function '${forbidden[1]}'`); - - // ── I3: no SETTINGS/SET/FORMAT in the emitted statement. - if (/\bSETTINGS\b/i.test(sql) || /\bFORMAT\b/i.test(sql)) { - fail("emitted statement contains a SETTINGS/FORMAT clause"); - } -}; diff --git a/packages/core/src/utils/deterministic-id.ts b/packages/core/src/utils/deterministic-id.ts index 89f54961b..203561221 100644 --- a/packages/core/src/utils/deterministic-id.ts +++ b/packages/core/src/utils/deterministic-id.ts @@ -18,13 +18,13 @@ * The triple `(anchor, eventName, personId)` is injective across every revenue * mapper, so the derived id is stable across retries / re-dispatch and collides * for exactly the duplicates an at-least-once queue may produce — the property - * the ClickHouse `(project_id, event_id)` pre-insert dedup relies on. This + * the portable `(project_id, event_id)` uniqueness constraint relies on. This * replaces the previous `generateId("analyticsEvent")` (a fresh random id per * call), which was the one piece genuinely incompatible with at-least-once * delivery. * - * `event_id` is an unbounded ClickHouse `String`, so a structured, legible - * deterministic string is used rather than a fixed-width hash — that keeps the + * A structured, legible deterministic string is used rather than a fixed-width + * hash — that keeps the * 17 sync mapper builders synchronous (no need to thread an async hash) and * keeps ids debuggable. * diff --git a/packages/core/test/_testing/CoreIntegrationTestHarness.ts b/packages/core/test/_testing/CoreIntegrationTestHarness.ts index 3ec4f0d14..98731b00f 100644 --- a/packages/core/test/_testing/CoreIntegrationTestHarness.ts +++ b/packages/core/test/_testing/CoreIntegrationTestHarness.ts @@ -1,4 +1,3 @@ -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; import { AuthSession } from "@voidhash/core/domain/auth/Auth"; import { generateId } from "@voidhash/core/utils"; import { @@ -20,7 +19,7 @@ import type {} from "./provided-context.d.ts"; /** * The common services every core integration test gets for free. The harness - * provides infra (`Db`, `ClickhouseWebClient`) plus the cross-cutting support + * provides PostgreSQL plus the cross-cutting support * services (`ProjectSchemaCache` stub, database-backed `AuditLogPort`, * `SchemaCacheInvalidationService`) that most feature-service layers depend on. * A test still provides its own service-under-test layer (e.g. @@ -30,7 +29,6 @@ import type {} from "./provided-context.d.ts"; */ type HarnessServices = | Db - | ClickhouseWebClient.ClickhouseWebClient | ProjectSchemaCache | AuditLogPort | PublicFileStore @@ -124,15 +122,12 @@ const AuditLogPortTestLive: Layer.Layer = Layer.effect( /** * Build the live infra + support layer from the shared - * {@link CoreTestConnections}. Every credential is real (Db/Clickhouse over - * the network); only the schema cache is a stub. + * {@link CoreTestConnections}. PostgreSQL is real; only the schema cache is a stub. */ const makeHarnessLayer = (tc: CoreTestConnections): Layer.Layer => { const DbLive: Layer.Layer = Db.layer(tc.db); - const ClickhouseLive = ClickhouseWebClient.layer(tc.clickhouse).pipe(Layer.orDie); const InfraLayer = Layer.mergeAll( DbLive, - ClickhouseLive, ProjectSchemaCacheStubLive, PublicFileStoreStubLive, ); diff --git a/packages/core/test/_testing/CoreTestConnections.ts b/packages/core/test/_testing/CoreTestConnections.ts index 07a4124da..8ffaf83fa 100644 --- a/packages/core/test/_testing/CoreTestConnections.ts +++ b/packages/core/test/_testing/CoreTestConnections.ts @@ -15,12 +15,6 @@ export interface CoreTestConnections { readonly password: string; readonly databaseName: string; }; - readonly clickhouse: { - readonly url: string; - readonly username: string; - readonly password: string; - readonly database: string; - }; } /** @@ -48,10 +42,4 @@ export const coreTestConnectionsFromEnv = ( password: env.DATABASE_PASSWORD ?? "password", databaseName: env.DATABASE_NAME ?? "voidhash", }, - clickhouse: { - url: env.CLICKHOUSE_URL ?? "http://127.0.0.1:8123", - username: env.CLICKHOUSE_USERNAME ?? "voidhash_app", - password: env.CLICKHOUSE_PASSWORD ?? "password", - database: env.CLICKHOUSE_DATABASE ?? "voidhash", - }, }); diff --git a/packages/core/test/domain/analytics/AnalyticsEvent.test.ts b/packages/core/test/domain/analytics/AnalyticsEvent.test.ts new file mode 100644 index 000000000..854b4f06c --- /dev/null +++ b/packages/core/test/domain/analytics/AnalyticsEvent.test.ts @@ -0,0 +1,135 @@ +import { + analyticsEventFromCapture, + analyticsEventFromHostedProcessed, + analyticsEventFromInternal, + COMMUNITY_CAPTURE_EVENT_NAMES, + isCommunityCaptureEventName, +} from "@voidhash/core/domain/analytics/AnalyticsEvent"; +import type { InternalAnalyticsEvent } from "@voidhash/core/domain/internalAnalytics/InternalAnalyticsEvents"; +import { DateTime } from "effect"; +import { describe, expect, it } from "vitest"; + +const occurredAt = DateTime.toDateUtc(DateTime.makeUnsafe("2026-08-01T12:34:56.789Z")); + +const revenueEvent = (): InternalAnalyticsEvent => ({ + context: { sdk: "react-native" }, + distinctId: "customer-1", + eventId: "revenue-event-1", + eventName: "$purchase.completed", + occurredAt, + organizationId: "org-1", + personId: "person-1", + projectId: "project-1", + properties: { + amount: 1299, + amountUsd: 1299, + currency: "USD", + paymentProviderConfigurationId: "configuration-1", + paymentProviderConfigurationProductId: "configuration-product-1", + providerEnvironment: 1, + providerEventType: "purchase", + providerId: "app-store", + providerSubscriptionId: null, + providerTransactionId: "transaction-1", + providerWebhookNotificationId: null, + source: "sdk", + }, + token: "internal", + transactionId: "transaction-1", +}); + +describe("shared analytics event contract", () => { + it("defines the complete Community SDK allow-list", () => { + expect(COMMUNITY_CAPTURE_EVENT_NAMES).toEqual([ + "$app_installed", + "$app_updated", + "$app_opened", + "$app_backgrounded", + "$app_became_active", + "$sign_out", + ]); + expect(isCommunityCaptureEventName("$app_opened")).toBe(true); + expect(isCommunityCaptureEventName("checkout_started")).toBe(false); + expect(isCommunityCaptureEventName("$identify")).toBe(false); + expect(isCommunityCaptureEventName("$purchase.completed")).toBe(false); + }); + + it("keeps capture IDs and event IDs stable across SDK retries", () => { + const input = { + event: { + uuid: "sdk-event-1", + event: "$app_opened", + context: { locale: "en-US" }, + properties: { $app_version: "2.0.0" }, + distinct_id: "device-1", + }, + organizationId: "org-1", + projectId: "project-1", + receivedAt: occurredAt, + requestId: "request-1", + requestPath: "/i/v1/capture", + sentAt: occurredAt, + token: "vh_pk_test", + }; + + const first = analyticsEventFromCapture(input); + const retry = analyticsEventFromCapture({ ...input, requestId: "request-2" }); + + expect(first.eventId).toBe("sdk-event-1"); + expect(first.captureId).toBe("capture_sdk-event-1"); + expect(retry.eventId).toBe(first.eventId); + expect(retry.captureId).toBe(first.captureId); + expect(first.identityMode).toBe("personless"); + }); + + it("maps trusted revenue and hosted processed events to the same semantics", () => { + const portable = analyticsEventFromInternal(revenueEvent(), occurredAt); + const hosted = analyticsEventFromHostedProcessed({ + captureId: portable.captureId, + context: portable.context, + distinctId: portable.distinctId, + event: portable.eventName, + eventTimestamp: portable.eventTimestamp.toISOString(), + identity: { + distinctId: portable.distinctId, + mode: portable.identityMode, + personId: portable.personId ?? undefined, + }, + organizationId: portable.organizationId, + processedAt: portable.processedAt.toISOString(), + processedEventId: portable.eventId, + projectId: portable.projectId, + properties: portable.properties, + request: { path: portable.requestPath ?? undefined, requestId: portable.requestId }, + routing: { sourceTopic: portable.sourceTopic }, + token: portable.token, + }); + + expect(hosted).toEqual(portable); + }); + + it("normalizes the hosted capture wrapper to the stored SDK properties", () => { + const hosted = analyticsEventFromHostedProcessed({ + captureId: "capture-sdk-1", + context: {}, + distinctId: "device-1", + event: "$app_opened", + eventTimestamp: occurredAt.toISOString(), + identity: { distinctId: "device-1", mode: "personless" }, + organizationId: "org-1", + processedAt: occurredAt.toISOString(), + processedEventId: "sdk-1", + projectId: "project-1", + properties: { + distinctId: "device-1", + properties: { $app_version: "2.0.0" }, + $process_person_profile: false, + }, + request: { requestId: "request-1" }, + routing: { sourceTopic: "capture.v1" }, + token: "vh_pk_test", + }); + + expect(hosted.properties).toEqual({ $app_version: "2.0.0" }); + }); +}); diff --git a/packages/core/test/domain/analytics/custom-insights.test.ts b/packages/core/test/domain/analytics/custom-insights.test.ts deleted file mode 100644 index f823def68..000000000 --- a/packages/core/test/domain/analytics/custom-insights.test.ts +++ /dev/null @@ -1,648 +0,0 @@ -import { constant } from "@voidhash/lib/lang"; -import { CustomAnalyticsInsightQuery } from "@voidhash/rpc"; -import { DateTime, Effect, Schema } from "effect"; - -import { describe, expect, it } from "../../../src/testing/effect-vitest.ts"; -import { InvalidAnalyticsQueryError } from "../../../src/domain/analytics/Analytics.ts"; -import { - alignTrendsComparisonPoints, - applyTrendsPresentation, - buildFunnelStepResults, - buildLifecycleSeries, - buildPathsLinkResults, - buildRetentionCohortResults, - buildStickinessBuckets, - buildTrendsFormulaSeries, - countStickinessIntervals, - fillTrendsSeriesPoints, - resolveTrendsComparisonTimeRange, - validateExecutableFunnelsDefinition, - validateExecutableLifecycleDefinition, - validateExecutablePathsDefinition, - validateExecutableRetentionDefinition, - validateExecutableStickinessDefinition, - validateExecutableTrendsDefinition, -} from "../../../src/services/analytics/CustomAnalyticsService.ts"; - -/** - * Builds a fixed UTC `Date` from an ISO string without touching the `Date` - * global, so these fixtures stay deterministic and lint-clean. - */ -const at = (iso: string): Date => DateTime.toDateUtc(DateTime.makeUnsafe(iso)); - -const eventSeries = { - aggregation: constant("unique_users"), - eventNames: constant(["$screen"]), - key: "A", -}; - -const timeRange = { preset: constant("last_7d") }; - -const fixtures = constant([ - { - comparison: "previous_period", - display: "line", - granularity: "day", - kind: "trends", - series: [eventSeries], - timeRange, - }, - { - conversionWindowSeconds: 86_400, - kind: "funnels", - order: "sequential", - steps: [ - { eventNames: ["paywall_viewed"], key: "A" }, - { eventNames: ["purchase_completed"], key: "B" }, - ], - timeRange, - }, - { - kind: "retention", - period: "week", - returning: { ...eventSeries, eventNames: ["session_started"], key: "B" }, - start: eventSeries, - timeRange, - }, - { - eventNames: ["$screen", "button_pressed"], - kind: "paths", - maxDepth: 8, - timeRange, - }, - { - interval: "day", - kind: "stickiness", - series: [eventSeries], - timeRange, - }, - { - granularity: "week", - kind: "lifecycle", - series: eventSeries, - timeRange, - }, -]); - -describe("CustomAnalyticsInsightQuery", () => { - it.effect("decodes every supported insight family", () => - Effect.gen(function* () { - for (const fixture of fixtures) { - const decoded = yield* Schema.decodeUnknownEffect(CustomAnalyticsInsightQuery)(fixture); - expect(decoded.kind).toBe(fixture.kind); - } - }), - ); - - it.effect("rejects a trends definition without a series", () => - Effect.gen(function* () { - const error = yield* Schema.decodeUnknownEffect(CustomAnalyticsInsightQuery)({ - display: "line", - granularity: "day", - kind: "trends", - series: [], - timeRange, - }).pipe(Effect.flip); - expect(error).toBeDefined(); - }), - ); -}); - -describe("validateExecutableTrendsDefinition", () => { - it.effect("accepts the first live trends subset", () => - Effect.gen(function* () { - const definition = yield* validateExecutableTrendsDefinition(fixtures[0]); - expect(definition.kind).toBe("trends"); - expect(definition.comparison).toBe("previous_period"); - expect(definition.series[0].aggregation).toBe("unique_users"); - }), - ); - - it.effect("fails with a typed error for a future query family", () => - Effect.gen(function* () { - const error = yield* validateExecutableTrendsDefinition(fixtures[1]).pipe(Effect.flip); - expect(error).toBeInstanceOf(InvalidAnalyticsQueryError); - expect(error.message).toContain("funnels"); - }), - ); - - it.effect("accepts custom property filters and breakdowns", () => - Effect.gen(function* () { - const definition = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - breakdown: { field: "event.properties.country", limit: 8 }, - series: [ - { - ...eventSeries, - filters: { - field: "event.properties.plan", - op: "eq", - type: "predicate", - value: "pro", - }, - }, - ], - }); - expect(definition.breakdown?.field).toBe("event.properties.country"); - expect(definition.series[0].filters).toBeDefined(); - }), - ); - - it.effect("rejects fields that cannot be lowered safely", () => - Effect.gen(function* () { - const error = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - breakdown: { field: "context.device.secret" }, - }).pipe(Effect.flip); - expect(error).toBeInstanceOf(InvalidAnalyticsQueryError); - expect(error.message).toContain("Unsupported custom analytics field"); - }), - ); - - it("resolves and aligns an equal previous-period window", () => { - const current = { - end: at("2026-07-13T12:00:00.000Z"), - start: at("2026-07-06T12:00:00.000Z"), - }; - const comparison = resolveTrendsComparisonTimeRange("previous_period", current); - - expect(comparison).toEqual({ - end: at("2026-07-06T11:59:59.000Z"), - start: at("2026-06-29T12:00:00.000Z"), - }); - expect( - alignTrendsComparisonPoints( - [{ timestamp: at("2026-06-30T00:00:00.000Z"), value: 12 }], - current, - comparison, - "day", - ), - ).toEqual([{ timestamp: at("2026-07-07T00:00:00.000Z"), value: 12 }]); - }); - - it("clamps leap day when resolving a previous-year comparison", () => { - const comparison = resolveTrendsComparisonTimeRange("previous_year", { - end: at("2024-03-01T08:30:00.000Z"), - start: at("2024-02-29T08:30:00.000Z"), - }); - - expect(comparison).toEqual({ - end: at("2023-03-01T08:30:00.000Z"), - start: at("2023-02-28T08:30:00.000Z"), - }); - }); - - it("aligns weekly comparisons by bucket position and fills sparse buckets", () => { - const current = { - end: at("2026-07-31T00:00:00.000Z"), - start: at("2026-07-01T00:00:00.000Z"), - }; - const comparison = resolveTrendsComparisonTimeRange("previous_period", current); - expect( - alignTrendsComparisonPoints( - [{ timestamp: at("2026-06-08T00:00:00.000Z"), value: 7 }], - current, - comparison, - "week", - ), - ).toEqual([{ timestamp: at("2026-07-06T00:00:00.000Z"), value: 7 }]); - - expect( - fillTrendsSeriesPoints( - [{ timestamp: at("2026-07-07T00:00:00.000Z"), value: 3 }], - { - end: at("2026-07-08T12:00:00.000Z"), - start: at("2026-07-06T12:00:00.000Z"), - }, - "day", - ), - ).toEqual([ - { timestamp: at("2026-07-06T00:00:00.000Z"), value: 0 }, - { timestamp: at("2026-07-07T00:00:00.000Z"), value: 3 }, - { timestamp: at("2026-07-08T00:00:00.000Z"), value: 0 }, - ]); - }); - - it("smooths, accumulates, and removes weekend buckets in presentation order", () => { - const points = [ - { timestamp: at("2026-07-10T00:00:00.000Z"), value: 2 }, - { timestamp: at("2026-07-11T00:00:00.000Z"), value: 4 }, - { timestamp: at("2026-07-12T00:00:00.000Z"), value: 6 }, - { timestamp: at("2026-07-13T00:00:00.000Z"), value: 8 }, - ]; - - expect( - applyTrendsPresentation(points, { - cumulative: true, - hideWeekends: true, - smoothingWindow: 2, - }), - ).toEqual([ - { timestamp: at("2026-07-10T00:00:00.000Z"), value: 2 }, - { timestamp: at("2026-07-13T00:00:00.000Z"), value: 17 }, - ]); - }); - - it.effect("rejects time-series presentation options on incompatible Trends definitions", () => - Effect.gen(function* () { - const smoothingError = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - granularity: "week", - smoothingWindow: 7, - }).pipe(Effect.flip); - expect(smoothingError.message).toContain("whole-day window"); - - const numberError = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - cumulative: true, - display: "number", - }).pipe(Effect.flip); - expect(numberError.message).toContain("do not support time-series"); - }), - ); - - it.effect("validates formula syntax and series references", () => - Effect.gen(function* () { - const definition = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - formulas: [{ expression: "(A + 2) ** 2", key: "calculated" }], - }); - expect(definition.formulas?.[0]?.expression).toBe("(A + 2) ** 2"); - - const unknownSeries = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - formulas: [{ expression: "A / B", key: "calculated" }], - }).pipe(Effect.flip); - expect(unknownSeries.message).toContain("unknown series: b"); - - const unsafeSyntax = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - formulas: [{ expression: "globalThis.process", key: "calculated" }], - }).pipe(Effect.flip); - expect(unsafeSyntax.message).toContain("Unexpected character"); - }), - ); - - it.effect("requires a safe numeric event property for property aggregations", () => - Effect.gen(function* () { - const definition = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - series: [{ ...eventSeries, aggregation: "property_average", mathProperty: "duration_ms" }], - }); - expect(definition.series[0].mathProperty).toBe("duration_ms"); - - const missingProperty = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - series: [{ ...eventSeries, aggregation: "property_sum" }], - }).pipe(Effect.flip); - expect(missingProperty.message).toContain("requires an event property"); - - const unusedProperty = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - series: [{ ...eventSeries, mathProperty: "duration_ms" }], - }).pipe(Effect.flip); - expect(unusedProperty.message).toContain("does not use an event property"); - }), - ); - - it.effect("builds formula series with sparse buckets and aligned comparisons", () => - Effect.gen(function* () { - const definition = yield* validateExecutableTrendsDefinition({ - ...fixtures[0], - formulas: [{ expression: "A / B * 100", key: "rate", label: "Activation rate" }], - series: [eventSeries, { ...eventSeries, eventNames: ["activated"], key: "B" }], - }); - const first = at("2026-07-06T00:00:00.000Z"); - const second = at("2026-07-07T00:00:00.000Z"); - const result = yield* buildTrendsFormulaSeries(definition, [ - { - comparison: "current", - key: "A", - label: "A", - points: [ - { timestamp: first, value: 10 }, - { timestamp: second, value: 5 }, - ], - }, - { - comparison: "current", - key: "B", - label: "B", - points: [{ timestamp: first, value: 20 }], - }, - { - comparison: "previous_period", - key: "A:comparison:previous_period", - label: "A (previous period)", - points: [{ timestamp: first, value: 4 }], - }, - { - comparison: "previous_period", - key: "B:comparison:previous_period", - label: "B (previous period)", - points: [{ timestamp: first, value: 8 }], - }, - ]); - - expect(result).toHaveLength(2); - expect(result[0]).toMatchObject({ - comparison: "current", - key: "rate", - label: "Activation rate", - points: [ - { timestamp: first, value: 50 }, - { timestamp: second, value: 0 }, - ], - }); - expect(result[1]).toMatchObject({ - comparison: "previous_period", - key: "rate:comparison:previous_period", - label: "Activation rate (previous period)", - points: [{ timestamp: first, value: 50 }], - }); - }), - ); -}); - -describe("executable funnels", () => { - it.effect("accepts sequential, strict, and any-order funnels", () => - Effect.gen(function* () { - for (const order of constant(["sequential", "strict", "any"])) { - const definition = yield* validateExecutableFunnelsDefinition({ ...fixtures[1], order }); - expect(definition.order).toBe(order); - } - }), - ); - - it.effect("accepts a validated breakdown attribution step", () => - Effect.gen(function* () { - const definition = yield* validateExecutableFunnelsDefinition({ - ...fixtures[1], - breakdown: { field: "event.properties.platform", limit: 5 }, - breakdownAttributionStep: 2, - }); - expect(definition.breakdown?.field).toBe("event.properties.platform"); - expect(definition.breakdownAttributionStep).toBe(2); - - const error = yield* validateExecutableFunnelsDefinition({ - ...fixtures[1], - breakdown: { field: "event.properties.platform" }, - breakdownAttributionStep: 3, - }).pipe(Effect.flip); - expect(error.message).toContain("existing step"); - }), - ); - - it.effect("rejects one-step funnels and unsafe conversion windows", () => - Effect.gen(function* () { - const oneStepError = yield* validateExecutableFunnelsDefinition({ - ...fixtures[1], - steps: [fixtures[1].steps[0]], - }).pipe(Effect.flip); - expect(oneStepError).toBeInstanceOf(InvalidAnalyticsQueryError); - - const windowError = yield* validateExecutableFunnelsDefinition({ - ...fixtures[1], - conversionWindowSeconds: 31_536_001, - }).pipe(Effect.flip); - expect(windowError.message).toContain("365 days"); - }), - ); - - it.effect("derives conversion and drop-off metrics from reach counts", () => - Effect.gen(function* () { - const definition = yield* validateExecutableFunnelsDefinition(fixtures[1]); - const steps = buildFunnelStepResults(definition, [100, 64]); - - expect(steps[0]).toMatchObject({ conversionRate: 1, count: 100, dropoffCount: 0 }); - expect(steps[1]).toMatchObject({ - conversionRate: 0.64, - count: 64, - dropoffCount: 36, - dropoffRate: 0.36, - }); - }), - ); -}); - -describe("executable retention", () => { - it.effect("accepts recurring and first-time cohort definitions", () => - Effect.gen(function* () { - for (const retentionType of constant(["recurring", "first_time"])) { - const definition = yield* validateExecutableRetentionDefinition({ - ...fixtures[2], - cumulative: true, - intervals: 8, - reference: "cohort", - retentionType, - }); - expect(definition.retentionType).toBe(retentionType); - } - }), - ); - - it.effect("rejects invalid interval counts", () => - Effect.gen(function* () { - const error = yield* validateExecutableRetentionDefinition({ - ...fixtures[2], - intervals: 25, - }).pipe(Effect.flip); - expect(error).toBeInstanceOf(InvalidAnalyticsQueryError); - expect(error.message).toContain("24 intervals"); - }), - ); - - it.effect("calculates cohort-relative and previous-period rates", () => - Effect.gen(function* () { - const base = yield* validateExecutableRetentionDefinition({ ...fixtures[2], intervals: 3 }); - const raw = [ - { - cohortSize: 100, - cohortStart: at("2026-07-01T00:00:00.000Z"), - counts: [80, 40, 20], - }, - ]; - expect(buildRetentionCohortResults(base, raw)[0]?.cells.map((cell) => cell.rate)).toEqual([ - 0.8, 0.4, 0.2, - ]); - expect( - buildRetentionCohortResults({ ...base, reference: "previous" }, raw)[0]?.cells.map( - (cell) => cell.rate, - ), - ).toEqual([0.8, 0.5, 0.5]); - }), - ); -}); - -describe("executable paths", () => { - it.effect("accepts mobile screen paths with session and density controls", () => - Effect.gen(function* () { - const definition = yield* validateExecutablePathsDefinition({ - ...fixtures[3], - collapseRepeated: true, - edgeLimit: 80, - filters: { - field: "event.properties.platform", - op: "eq", - type: "predicate", - value: "ios", - }, - pathItem: "screen_name", - sessionGapSeconds: 1_800, - }); - expect(definition.pathItem).toBe("screen_name"); - expect(definition.sessionGapSeconds).toBe(1_800); - }), - ); - - it.effect("rejects empty endpoints and inverted edge-density controls", () => - Effect.gen(function* () { - const endpointError = yield* validateExecutablePathsDefinition({ - ...fixtures[3], - startEventName: "", - }).pipe(Effect.flip); - expect(endpointError).toBeInstanceOf(InvalidAnalyticsQueryError); - expect(endpointError.message).toContain("cannot be empty"); - - const densityError = yield* validateExecutablePathsDefinition({ - ...fixtures[3], - maxEdgeCount: 4, - minEdgeCount: 5, - }).pipe(Effect.flip); - expect(densityError.message).toContain("minimum link count"); - }), - ); - - it("normalizes path links without changing transition-frequency semantics", () => { - expect( - buildPathsLinkResults([ - { - averageTransitionSeconds: 12.5, - count: 7, - source: "Home", - sourceStep: 1, - target: "Paywall", - targetStep: 2, - }, - ]), - ).toEqual([ - { - averageTransitionSeconds: 12.5, - count: 7, - source: "Home", - sourceStep: 1, - target: "Paywall", - targetStep: 2, - }, - ]); - }); -}); - -describe("executable stickiness", () => { - it.effect("accepts exact and cumulative frequency definitions", () => - Effect.gen(function* () { - for (const computation of constant(["exact", "cumulative"])) { - const definition = yield* validateExecutableStickinessDefinition({ - ...fixtures[4], - computation, - occurrenceCriteria: { operator: "gte", value: 2 }, - }); - expect(definition.computation).toBe(computation); - } - }), - ); - - it.effect("rejects more than eight series", () => - Effect.gen(function* () { - const error = yield* validateExecutableStickinessDefinition({ - ...fixtures[4], - series: [ - { ...eventSeries, key: "0" }, - ...Array.from({ length: 8 }, (_, index) => ({ - ...eventSeries, - key: String(index + 1), - })), - ], - }).pipe(Effect.flip); - expect(error).toBeInstanceOf(InvalidAnalyticsQueryError); - expect(error.message).toContain("8 series"); - }), - ); - - it("fills exact gaps and builds at-least-N cumulative buckets", () => { - const raw = [ - { count: 5, intervals: 1 }, - { count: 2, intervals: 3 }, - ]; - expect(buildStickinessBuckets(raw, "exact", 4)).toEqual([ - { count: 5, intervals: 1 }, - { count: 0, intervals: 2 }, - { count: 2, intervals: 3 }, - { count: 0, intervals: 4 }, - ]); - expect(buildStickinessBuckets(raw, "cumulative", 4)).toEqual([ - { count: 7, intervals: 1 }, - { count: 2, intervals: 2 }, - { count: 2, intervals: 3 }, - { count: 0, intervals: 4 }, - ]); - }); - - it("counts inclusive hourly, weekly, and monthly buckets", () => { - expect( - countStickinessIntervals(at("2026-07-13T10:30:00Z"), at("2026-07-13T12:00:00Z"), "hour"), - ).toBe(3); - expect( - countStickinessIntervals(at("2026-07-12T00:00:00Z"), at("2026-07-13T00:00:00Z"), "week"), - ).toBe(2); - expect( - countStickinessIntervals(at("2025-12-31T00:00:00Z"), at("2026-02-01T00:00:00Z"), "month"), - ).toBe(3); - }); - - it.effect("rejects event-count aggregation", () => - Effect.gen(function* () { - const error = yield* validateExecutableStickinessDefinition({ - ...fixtures[4], - series: [{ ...eventSeries, aggregation: "total_events" }], - }).pipe(Effect.flip); - expect(error.message).toContain("unique users"); - }), - ); -}); - -describe("executable lifecycle", () => { - it.effect("accepts selected lifecycle statuses and rejects event aggregation", () => - Effect.gen(function* () { - const definition = yield* validateExecutableLifecycleDefinition({ - ...fixtures[5], - display: "stacked_area", - statuses: ["new", "returning", "dormant"], - }); - expect(definition.statuses).toEqual(["new", "returning", "dormant"]); - - const error = yield* validateExecutableLifecycleDefinition({ - ...fixtures[5], - series: { ...eventSeries, aggregation: "total_events" }, - }).pipe(Effect.flip); - expect(error.message).toContain("unique users"); - }), - ); - - it("fills every selected status across the inclusive range", () => { - const series = buildLifecycleSeries( - [ - { count: 2, status: "returning", timestamp: at("2026-07-02T00:00:00Z") }, - { count: 1, status: "dormant", timestamp: at("2026-07-03T00:00:00Z") }, - ], - ["returning", "dormant"], - at("2026-07-01T12:00:00Z"), - at("2026-07-03T18:00:00Z"), - "day", - ); - expect(series.map((item) => [item.status, item.points.map((point) => point.count)])).toEqual([ - ["returning", [0, 2, 0]], - ["dormant", [0, 0, 1]], - ]); - }); -}); diff --git a/packages/core/test/domain/analyticsIngest/AnalyticsIngest.test.ts b/packages/core/test/domain/analyticsIngest/AnalyticsIngest.test.ts deleted file mode 100644 index 9fcad40e6..000000000 --- a/packages/core/test/domain/analyticsIngest/AnalyticsIngest.test.ts +++ /dev/null @@ -1,894 +0,0 @@ -import { DateTime, Schema } from "effect"; -import { describe, expect, it } from "vite-plus/test"; - -import { - ANONYMOUS_DISTINCT_ID_PREFIX, - buildAnalyticsWriterPlan, - buildDlqEvent, - computeCutoffIso, - extractInnerProperties, - extractPreviousDistinctId, - makeCapturedEventFromInternalAnalyticsEvent, - makeSnapshotResources, - parsePersonTraits, - sanitizeIdentifier, - toClickhouseTimestamp, - toFlag, - toPendingOverrideRow, - toPersonIdentityRow, - toPersonRow, - toProcessedEventRow, - validateBuiltInProcessorRules, - type AnalyticsWriterMessageType, - type CapturedEventV1Type, - type EventProcessorDlqV1, - type ProcessedEventV2Type, - type ProcessorPersonEventV1Type, - type ProcessorPersonIdentityEventV1Type, -} from "../../../src/domain/analyticsIngest/AnalyticsIngest.ts"; -import { REVENUE_TRUSTED_SOURCE_TOPIC } from "../../../src/domain/internalAnalytics/InternalAnalyticsEvents.ts"; - -// ============================================================================= -// Fixtures — const-returning builders, one fresh object per test. -// ============================================================================= - -/** Fixed instants: a `Date` built from an ISO string without touching globals. */ -const dateAt = (iso: string): Date => DateTime.toDateUtc(DateTime.makeUnsafe(iso)); - -/** JSON codecs standing in for `JSON.stringify` / `JSON.parse`. */ -const encodeJson = Schema.encodeSync(Schema.UnknownFromJsonString); -const decodeEnvelopeProperties = Schema.decodeSync( - Schema.fromJsonString( - Schema.Struct({ properties: Schema.Record(Schema.String, Schema.Unknown) }), - ), -); - -/** - * A minimal valid {@link CapturedEventV1Type}. The two structural blocks that - * the processor validation cares about — `routing` and `properties` — are - * overridable; everything else is filler that keeps the shape type-valid. - */ -const capturedEvent = ( - overrides: { - readonly event?: string; - readonly distinctId?: string; - readonly eventTimestamp?: string; - readonly properties?: CapturedEventV1Type["properties"]; - readonly routing?: Partial; - } = {}, -): CapturedEventV1Type => ({ - schemaVersion: 1, - captureId: "cap_1", - token: "tok_1", - organizationId: "org_1", - projectId: "proj_1", - event: overrides.event ?? "page_view", - distinctId: overrides.distinctId ?? "user_1", - eventTimestamp: overrides.eventTimestamp ?? "2026-01-01T00:00:00.000Z", - receivedAt: "2026-01-01T00:00:00.000Z", - properties: overrides.properties ?? {}, - context: {}, - rawPayload: {}, - request: { requestId: "req_1" }, - routing: { - routeClass: "main", - targetTopic: "main.topic", - isHistorical: false, - skipEnrichment: false, - ...overrides.routing, - }, -}); - -const processedEvent = (overrides: Partial = {}): ProcessedEventV2Type => ({ - captureId: "cap_1", - context: { ua: "test" }, - distinctId: "user_1", - event: "page_view", - eventTimestamp: "2026-01-01T12:34:56.789Z", - groups: [], - identity: { distinctId: "user_1", mode: "full", personId: "person_1" }, - organizationId: "org_1", - processedAt: "2026-01-02T01:02:03.004Z", - processedEventId: "ev_1", - projectId: "proj_1", - properties: { foo: "bar" }, - request: { requestId: "req_1", path: "/capture" }, - routing: { - lane: "main", - skipEnrichment: false, - sourceOffset: "42", - sourcePartition: 3, - sourceTopic: "main.topic", - }, - schemaVersion: 2, - token: "tok_1", - ...overrides, -}); - -const personEvent = ( - overrides: Partial = {}, -): ProcessorPersonEventV1Type => ({ - changedAt: "2026-01-01T00:00:00.000Z", - personId: "person_1", - isArchived: false, - projectId: "proj_1", - schemaVersion: 1, - traits: { plan: "pro" }, - version: 5, - ...overrides, -}); - -const personIdentityEvent = ( - overrides: Partial = {}, -): ProcessorPersonIdentityEventV1Type => ({ - changedAt: "2026-01-01T00:00:00.000Z", - personId: "person_1", - distinctId: "user_1", - isDeleted: false, - projectId: "proj_1", - schemaVersion: 1, - version: 1, - ...overrides, -}); - -const dlqInput = ( - overrides: Partial> = {}, -): Omit => ({ - failureClass: "schema_rejected", - failureMessage: "boom", - headers: { "x-foo": "bar" }, - lane: "main", - sourceOffset: "7", - sourcePartition: 1, - sourceTopic: "main.topic", - ...overrides, -}); - -// ============================================================================= -// parsePersonTraits -// ============================================================================= - -describe("parsePersonTraits", () => { - it("returns empty set and setOnce for empty properties", () => { - const result = parsePersonTraits({}); - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.value.set).toEqual({}); - expect(result.value.setOnce).toEqual({}); - } - }); - - it("parses a $set object", () => { - const result = parsePersonTraits({ $set: { plan: "pro" } }); - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.value.set).toEqual({ plan: "pro" }); - expect(result.value.setOnce).toEqual({}); - } - }); - - it("parses a $set_once object", () => { - const result = parsePersonTraits({ $set_once: { firstSeen: "today" } }); - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.value.setOnce).toEqual({ firstSeen: "today" }); - expect(result.value.set).toEqual({}); - } - }); - - it("parses both $set and $set_once", () => { - const result = parsePersonTraits({ - $set: { plan: "pro" }, - $set_once: { firstSeen: "today" }, - }); - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.value.set).toEqual({ plan: "pro" }); - expect(result.value.setOnce).toEqual({ firstSeen: "today" }); - } - }); - - it("rejects a non-object $set", () => { - const result = parsePersonTraits({ $set: "nope" }); - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.message).toBe("$set must be an object"); - } - }); - - it("rejects a non-object $set_once", () => { - const result = parsePersonTraits({ $set_once: ["nope"] }); - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.message).toBe("$set_once must be an object"); - } - }); -}); - -// ============================================================================= -// extractInnerProperties -// ============================================================================= - -describe("extractInnerProperties", () => { - it("extracts a nested 'properties' object", () => { - const inner = { a: 1 }; - expect(extractInnerProperties({ properties: inner, outer: 2 })).toBe(inner); - }); - - it("returns the input when 'properties' is missing", () => { - const input = { a: 1 }; - expect(extractInnerProperties(input)).toBe(input); - }); - - it("returns the input when 'properties' is not an object", () => { - const input = { properties: "string" }; - expect(extractInnerProperties(input)).toBe(input); - }); - - it("returns the input when 'properties' is an array (not a plain record)", () => { - const input = { properties: [1, 2, 3] }; - expect(extractInnerProperties(input)).toBe(input); - }); -}); - -// ============================================================================= -// validateBuiltInProcessorRules -// ============================================================================= - -describe("validateBuiltInProcessorRules", () => { - const now = dateAt("2026-01-10T00:00:00.000Z"); - - it("accepts a valid main-lane event", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ routing: { targetTopic: "main.topic" } }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBeUndefined(); - }); - - it("rejects when the routing target does not match the source topic", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ routing: { targetTopic: "other.topic" } }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("captured event routing target does not match source topic"); - }); - - it("rejects a historical-lane event not marked isHistorical", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ - routing: { targetTopic: "main.topic", isHistorical: false }, - }), - historicalMinAgeHours: 24, - lane: "historical", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("historical topic requires isHistorical=true"); - }); - - it("rejects a main-lane event marked as historical", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ - routing: { targetTopic: "main.topic", isHistorical: true }, - }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("non-historical lane received a historical captured event"); - }); - - it("accepts a historical event older than the minimum age", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ - // 48h before `now`, minimum age 24h → old enough. - eventTimestamp: "2026-01-08T00:00:00.000Z", - routing: { targetTopic: "main.topic", isHistorical: true }, - }), - historicalMinAgeHours: 24, - lane: "historical", - now, - sourceTopic: "main.topic", - }); - expect(result).toBeUndefined(); - }); - - it("rejects a historical event newer than the minimum age", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ - // 1h before `now`, minimum age 24h → too new. - eventTimestamp: "2026-01-09T23:00:00.000Z", - routing: { targetTopic: "main.topic", isHistorical: true }, - }), - historicalMinAgeHours: 24, - lane: "historical", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("historical event is newer than the configured minimum age"); - }); - - it("rejects a non-object $set in properties", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ properties: { $set: "nope" } }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("$set must be an object"); - }); - - it("rejects a non-object $set_once in properties", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ properties: { $set_once: 42 } }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("$set_once must be an object"); - }); - - it("rejects a non-boolean $process_person_profile", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ properties: { $process_person_profile: "yes" } }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("$process_person_profile must be a boolean"); - }); - - it("rejects an $identify without $previous_distinct_id", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ event: "$identify", properties: {} }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("$identify requires properties.$previous_distinct_id"); - }); - - it("rejects an $identify with an empty $previous_distinct_id", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ - event: "$identify", - properties: { $previous_distinct_id: "" }, - }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("$identify requires properties.$previous_distinct_id"); - }); - - it("rejects an $identify whose target distinct id uses the anonymous prefix", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ - event: "$identify", - distinctId: `${ANONYMOUS_DISTINCT_ID_PREFIX}abc`, - properties: { $previous_distinct_id: "prev_1" }, - }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBe("$identify target distinct id cannot use the anonymous prefix"); - }); - - it("accepts a valid $identify (previous id present, non-anonymous target)", () => { - const result = validateBuiltInProcessorRules({ - capturedEvent: capturedEvent({ - event: "$identify", - distinctId: "user_1", - properties: { $previous_distinct_id: "prev_1" }, - }), - historicalMinAgeHours: 24, - lane: "main", - now, - sourceTopic: "main.topic", - }); - expect(result).toBeUndefined(); - }); -}); - -// ============================================================================= -// buildDlqEvent -// ============================================================================= - -describe("buildDlqEvent", () => { - it("stamps a failedAt ISO timestamp and a schemaVersion of 1", () => { - const event = buildDlqEvent(dlqInput()); - expect(event.schemaVersion).toBe(1); - // failedAt round-trips through Date without losing precision. - expect(dateAt(event.failedAt).toISOString()).toBe(event.failedAt); - }); - - it("generates a unique UUID failureId per call", () => { - const a = buildDlqEvent(dlqInput()); - const b = buildDlqEvent(dlqInput()); - expect(a.failureId).toMatch(/^[0-9a-f-]{36}$/i); - expect(a.failureId).not.toBe(b.failureId); - }); - - it("includes the optional fields when provided", () => { - const event = buildDlqEvent( - dlqInput({ - captureId: "cap_x", - distinctId: "user_x", - projectId: "proj_x", - rawKey: "key_x", - rawValue: "value_x", - token: "tok_x", - }), - ); - expect(event.captureId).toBe("cap_x"); - expect(event.distinctId).toBe("user_x"); - expect(event.projectId).toBe("proj_x"); - expect(event.rawKey).toBe("key_x"); - expect(event.rawValue).toBe("value_x"); - expect(event.token).toBe("tok_x"); - }); - - it("omits the optional fields when not provided", () => { - const event = buildDlqEvent(dlqInput()); - expect("captureId" in event).toBe(false); - expect("distinctId" in event).toBe(false); - expect("projectId" in event).toBe(false); - expect("rawKey" in event).toBe(false); - expect("rawValue" in event).toBe(false); - expect("token" in event).toBe(false); - }); - - it("carries the non-optional fields straight through", () => { - const event = buildDlqEvent(dlqInput({ failureClass: "policy_rejected" })); - expect(event.failureClass).toBe("policy_rejected"); - expect(event.failureMessage).toBe("boom"); - expect(event.headers).toEqual({ "x-foo": "bar" }); - expect(event.lane).toBe("main"); - expect(event.sourceOffset).toBe("7"); - expect(event.sourcePartition).toBe(1); - expect(event.sourceTopic).toBe("main.topic"); - }); -}); - -// ============================================================================= -// toFlag / toClickhouseTimestamp / extractPreviousDistinctId -// ============================================================================= - -describe("toFlag", () => { - it("converts true to 1", () => { - expect(toFlag(true)).toBe(1); - }); - - it("converts false to 0", () => { - expect(toFlag(false)).toBe(0); - }); -}); - -describe("toClickhouseTimestamp", () => { - it("converts an ISO string to ClickHouse 'YYYY-MM-DD HH:MM:SS.mmm' UTC format", () => { - expect(toClickhouseTimestamp("2026-01-02T03:04:05.006Z")).toBe("2026-01-02 03:04:05.006"); - }); - - it("zero-pads each part", () => { - expect(toClickhouseTimestamp("2026-09-09T09:09:09.009Z")).toBe("2026-09-09 09:09:09.009"); - }); - - it("throws on an invalid timestamp", () => { - expect(() => toClickhouseTimestamp("not-a-date")).toThrow("Invalid timestamp: not-a-date"); - }); -}); - -describe("extractPreviousDistinctId", () => { - it("extracts from a nested $previous_distinct_id property", () => { - const event = processedEvent({ properties: { $previous_distinct_id: "prev_1" } }); - expect(extractPreviousDistinctId(event)).toBe("prev_1"); - }); - - it("extracts from a doubly-nested properties wrapper", () => { - const event = processedEvent({ - properties: { properties: { $previous_distinct_id: "prev_2" } }, - }); - expect(extractPreviousDistinctId(event)).toBe("prev_2"); - }); - - it("returns null when the field is missing", () => { - const event = processedEvent({ properties: { other: 1 } }); - expect(extractPreviousDistinctId(event)).toBeNull(); - }); - - it("returns null for an empty / whitespace-only string", () => { - const event = processedEvent({ properties: { $previous_distinct_id: " " } }); - expect(extractPreviousDistinctId(event)).toBeNull(); - }); -}); - -// ============================================================================= -// Row builders -// ============================================================================= - -describe("toProcessedEventRow", () => { - it("converts a ProcessedEventV2 to the ClickHouse row shape", () => { - const row = toProcessedEventRow(processedEvent()); - expect(row).toMatchObject({ - capture_id: "cap_1", - distinct_id: "user_1", - event_id: "ev_1", - event_name: "page_view", - event_ts: "2026-01-01 12:34:56.789", - identity_mode: "full", - organization_id: "org_1", - person_id: "person_1", - processed_ts: "2026-01-02 01:02:03.004", - project_id: "proj_1", - request_id: "req_1", - request_path: "/capture", - route_lane: "main", - schema_version: 2, - skip_enrichment: 0, - source_offset: "42", - source_partition: 3, - source_topic: "main.topic", - token: "tok_1", - }); - }); - - it("serializes context and properties as JSON strings", () => { - const row = toProcessedEventRow( - processedEvent({ context: { ua: "x" }, properties: { foo: "bar" } }), - ); - expect(row.context).toBe(encodeJson({ ua: "x" })); - expect(row.event_properties).toBe(encodeJson({ foo: "bar" })); - }); - - it("maps a missing person_id to null", () => { - const row = toProcessedEventRow( - processedEvent({ identity: { distinctId: "user_1", mode: "personless" } }), - ); - expect(row.person_id).toBeNull(); - expect(row.identity_mode).toBe("personless"); - }); - - it("defaults request_path to an empty string when absent", () => { - const row = toProcessedEventRow(processedEvent({ request: { requestId: "req_1" } })); - expect(row.request_path).toBe(""); - }); - - it("sets skip_enrichment to 1 when enrichment is skipped", () => { - const row = toProcessedEventRow( - processedEvent({ - routing: { - lane: "main", - skipEnrichment: true, - sourceOffset: "1", - sourcePartition: 0, - sourceTopic: "main.topic", - }, - }), - ); - expect(row.skip_enrichment).toBe(1); - }); -}); - -describe("toPersonRow", () => { - it("converts a person event to a row carrying the supplied organization_id", () => { - const row = toPersonRow(personEvent(), "org_resolved"); - expect(row).toMatchObject({ - changed_at: "2026-01-01 00:00:00.000", - organization_id: "org_resolved", - person_id: "person_1", - email: null, - is_archived: 0, - merged_into_person_id: null, - name: null, - primary_distinct_id: null, - project_id: "proj_1", - version: 5, - }); - }); - - it("encodes traits as a JSON string", () => { - const row = toPersonRow(personEvent({ traits: { plan: "pro" } }), "org_1"); - expect(row.traits).toBe(encodeJson({ plan: "pro" })); - }); - - it("flags archival and surfaces optional string fields", () => { - const row = toPersonRow( - personEvent({ - isArchived: true, - email: "a@b.com", - name: "Ada", - primaryDistinctId: "user_1", - mergedIntoPersonId: "person_2", - }), - "org_1", - ); - expect(row.is_archived).toBe(1); - expect(row.email).toBe("a@b.com"); - expect(row.name).toBe("Ada"); - expect(row.primary_distinct_id).toBe("user_1"); - expect(row.merged_into_person_id).toBe("person_2"); - }); -}); - -describe("toPersonIdentityRow", () => { - it("converts an identity event to a row", () => { - const row = toPersonIdentityRow( - personIdentityEvent({ previousDistinctId: "prev_1", isDeleted: true, version: 3 }), - "org_resolved", - ); - expect(row).toMatchObject({ - changed_at: "2026-01-01 00:00:00.000", - organization_id: "org_resolved", - person_id: "person_1", - distinct_id: "user_1", - is_deleted: 1, - previous_distinct_id: "prev_1", - project_id: "proj_1", - version: 3, - }); - }); - - it("maps an absent previous distinct id to null", () => { - const row = toPersonIdentityRow(personIdentityEvent(), "org_1"); - expect(row.previous_distinct_id).toBeNull(); - }); -}); - -describe("toPendingOverrideRow", () => { - it("builds the pending override with source and target distinct ids", () => { - const row = toPendingOverrideRow( - personIdentityEvent({ previousDistinctId: "prev_1", distinctId: "user_1", version: 2 }), - "org_resolved", - ); - expect(row).toMatchObject({ - changed_at: "2026-01-01 00:00:00.000", - organization_id: "org_resolved", - person_id: "person_1", - is_deleted: 0, - project_id: "proj_1", - source_distinct_id: "prev_1", - target_distinct_id: "user_1", - version: 2, - }); - }); - - it("defaults the source distinct id to an empty string when absent", () => { - const row = toPendingOverrideRow(personIdentityEvent(), "org_1"); - expect(row.source_distinct_id).toBe(""); - }); -}); - -// ============================================================================= -// buildAnalyticsWriterPlan -// ============================================================================= - -describe("buildAnalyticsWriterPlan", () => { - const resolveOrg = (projectId: string) => `org_for_${projectId}`; - - it("separates processed, person and identity messages into their lanes", () => { - const messages: ReadonlyArray = [ - { kind: "processed", messageId: "m1", value: processedEvent() }, - { kind: "person", messageId: "m2", value: personEvent() }, - { - kind: "person-distinct-id", - messageId: "m3", - // version 0 + no previousDistinctId → no override rows. - value: personIdentityEvent({ version: 0 }), - }, - ]; - const plan = buildAnalyticsWriterPlan(messages, resolveOrg); - expect(plan.processedEventRows).toHaveLength(1); - expect(plan.personRows).toHaveLength(1); - expect(plan.personIdentityRows).toHaveLength(1); - expect(plan.personIdentityOverrideRows).toHaveLength(0); - expect(plan.personIdentityPendingOverrideRows).toHaveLength(0); - }); - - it("emits override + pending-override rows for a versioned identity change with a previous id", () => { - const messages: ReadonlyArray = [ - { - kind: "person-distinct-id", - messageId: "m1", - value: personIdentityEvent({ previousDistinctId: "prev_1", version: 1 }), - }, - ]; - const plan = buildAnalyticsWriterPlan(messages, resolveOrg); - expect(plan.personIdentityRows).toHaveLength(1); - expect(plan.personIdentityOverrideRows).toHaveLength(1); - expect(plan.personIdentityPendingOverrideRows).toHaveLength(1); - expect(plan.personIdentityPendingOverrideRows[0]).toMatchObject({ - source_distinct_id: "prev_1", - target_distinct_id: "user_1", - }); - }); - - it("does not emit override rows when version is 0 even with a previous id", () => { - const messages: ReadonlyArray = [ - { - kind: "person-distinct-id", - messageId: "m1", - value: personIdentityEvent({ previousDistinctId: "prev_1", version: 0 }), - }, - ]; - const plan = buildAnalyticsWriterPlan(messages, resolveOrg); - expect(plan.personIdentityOverrideRows).toHaveLength(0); - expect(plan.personIdentityPendingOverrideRows).toHaveLength(0); - }); - - it("resolves the organization id via the callback for person and identity rows", () => { - const messages: ReadonlyArray = [ - { kind: "person", messageId: "m1", value: personEvent({ projectId: "proj_z" }) }, - { - kind: "person-distinct-id", - messageId: "m2", - value: personIdentityEvent({ projectId: "proj_z" }), - }, - ]; - const plan = buildAnalyticsWriterPlan(messages, resolveOrg); - expect(plan.personRows[0]?.organization_id).toBe("org_for_proj_z"); - expect(plan.personIdentityRows[0]?.organization_id).toBe("org_for_proj_z"); - }); - - it("returns all-empty lanes for an empty message list", () => { - const plan = buildAnalyticsWriterPlan([], resolveOrg); - expect(plan.processedEventRows).toHaveLength(0); - expect(plan.personRows).toHaveLength(0); - expect(plan.personIdentityRows).toHaveLength(0); - expect(plan.personIdentityOverrideRows).toHaveLength(0); - expect(plan.personIdentityPendingOverrideRows).toHaveLength(0); - }); -}); - -// ============================================================================= -// makeCapturedEventFromInternalAnalyticsEvent -// ============================================================================= - -describe("makeCapturedEventFromInternalAnalyticsEvent", () => { - it("maps a trusted revenue event onto a CapturedEventV1 with a Resolved claim", () => { - const occurredAt = dateAt("2026-03-04T05:06:07.008Z"); - const captured = makeCapturedEventFromInternalAnalyticsEvent({ - eventName: "$purchase.completed", - eventId: "an_evt_k1:_purchase.completed:person_1", - distinctId: "dist_1", - occurredAt, - organizationId: "org_1", - personId: "person_1", - projectId: "proj_1", - token: "tok_1", - transactionId: "txn_1", - context: { sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC }, - properties: { - paymentProviderConfigurationId: "cfg_1", - paymentProviderConfigurationProductId: "prod_1", - providerEnvironment: 1, - providerEventType: "PURCHASE", - providerId: "appstore", - providerSubscriptionId: null, - providerTransactionId: null, - providerWebhookNotificationId: null, - source: "appstore", - grossAmountUsd: 1_100, - }, - }); - - expect(captured.schemaVersion).toBe(1); - // The deterministic id rides as clientEventId → becomes events_v2.event_id. - expect(captured.clientEventId).toBe("an_evt_k1:_purchase.completed:person_1"); - expect(captured.captureId).toBe("internal_an_evt_k1:_purchase.completed:person_1"); - expect(captured.event).toBe("$purchase.completed"); - expect(captured.distinctId).toBe("dist_1"); - expect(captured.organizationId).toBe("org_1"); - expect(captured.projectId).toBe("proj_1"); - expect(captured.token).toBe("tok_1"); - expect(captured.eventTimestamp).toBe(occurredAt.toISOString()); - // Server-trusted: Resolved claim + trusted topic + skipEnrichment. - expect(captured.identityClaim).toEqual({ - _tag: "Resolved", - distinctId: "dist_1", - personId: "person_1", - }); - expect(captured.trustClass).toBe("trusted-revenue"); - expect(captured.routing.targetTopic).toBe(REVENUE_TRUSTED_SOURCE_TOPIC); - expect(captured.routing.skipEnrichment).toBe(true); - expect(captured.routing.isHistorical).toBe(false); - expect(captured.routing.routeClass).toBe("main"); - }); - - it("coerces Date-valued properties to ISO strings so the envelope is JSON-clean", () => { - const transferredAt = dateAt("2026-03-04T05:06:07.008Z"); - const captured = makeCapturedEventFromInternalAnalyticsEvent({ - eventName: "$subscription.transferred_in", - eventId: "an_evt_kx:_subscription.transferred_in:person_to", - distinctId: "dist_to", - occurredAt: transferredAt, - organizationId: "org_1", - personId: "person_to", - projectId: "proj_1", - token: "tok_1", - transactionId: null, - properties: { - paymentProviderConfigurationId: "cfg_1", - paymentProviderConfigurationProductId: "prod_1", - providerEnvironment: 1, - providerId: "appstore", - source: "appstore", - providerEventType: "subscription.transferred", - subscriptionId: "sub_1", - fromDistinctId: "dist_from", - fromPersonId: "person_from", - toDistinctId: "dist_to", - toPersonId: "person_to", - transferMode: "transfer_to_new_owner", - transferReason: "appstore_restore", - transferredAt, - }, - }); - // The Date becomes its ISO string — no live Date object survives onto the - // wire `properties` (which permits only JSON primitives). - expect(captured.properties.transferredAt).toBe(transferredAt.toISOString()); - expect(decodeEnvelopeProperties(encodeJson(captured)).properties.transferredAt).toBe( - transferredAt.toISOString(), - ); - }); -}); - -// ============================================================================= -// Janitor snapshot helpers -// ============================================================================= - -describe("sanitizeIdentifier", () => { - it("replaces non-alphanumeric characters with underscores", () => { - expect(sanitizeIdentifier("a-b.c d")).toBe("a_b_c_d"); - }); - - it("keeps valid alphanumeric and underscore characters", () => { - expect(sanitizeIdentifier("Abc_123")).toBe("Abc_123"); - }); -}); - -describe("makeSnapshotResources", () => { - it("generates deterministic, dash-stripped names from a supplied runId", () => { - const resources = makeSnapshotResources("run-12-34"); - expect(resources.pendingOverrideSnapshotName).toBe( - "person_identity_pending_override_snapshot_run1234", - ); - expect(resources.pendingOverrideDictionaryName).toBe( - "person_identity_pending_override_dict_run1234", - ); - }); - - it("generates unique names across calls when no runId is supplied (UUID default)", () => { - const a = makeSnapshotResources(); - const b = makeSnapshotResources(); - expect(a.pendingOverrideSnapshotName).not.toBe(b.pendingOverrideSnapshotName); - // The generated suffix must be a safe SQL identifier (no dashes / specials). - expect(a.pendingOverrideSnapshotName).toMatch(/^[a-zA-Z0-9_]+$/); - }); -}); - -describe("computeCutoffIso", () => { - it("subtracts the safety window from now", () => { - const now = dateAt("2026-01-01T00:00:10.000Z"); - expect(computeCutoffIso({ now, safetyWindowSeconds: 10 })).toBe("2026-01-01T00:00:00.000Z"); - }); - - it("treats a zero window as 'now'", () => { - const now = dateAt("2026-01-01T00:00:00.000Z"); - expect(computeCutoffIso({ now, safetyWindowSeconds: 0 })).toBe(now.toISOString()); - }); -}); diff --git a/packages/core/test/integration-harness.integration.test.ts b/packages/core/test/integration-harness.integration.test.ts index 6e7e22d03..cbd960737 100644 --- a/packages/core/test/integration-harness.integration.test.ts +++ b/packages/core/test/integration-harness.integration.test.ts @@ -12,6 +12,5 @@ test( // expose credentials for the in-process harness layers. expect(output.testConnections).not.toBeNull(); expect(output.testConnections?.db.host).toBeDefined(); - expect(output.testConnections?.clickhouse.url).toBeDefined(); }), ); diff --git a/packages/core/test/runtime-context-types.ts b/packages/core/test/runtime-context-types.ts index 04316cb75..4729ca500 100644 --- a/packages/core/test/runtime-context-types.ts +++ b/packages/core/test/runtime-context-types.ts @@ -1,11 +1,6 @@ import type { Effect } from "effect"; import type { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; -import type { - PolicyCounterStoreShape, - PolicyStoreError, - RequestLimitCheck, -} from "../src/services/analyticsIngest/PolicyCounterStore.ts"; import type { QueueProducer, QueueProducerError, @@ -13,9 +8,6 @@ import type { declare const producer: QueueProducer; -/** Type-level fixture: this file is typechecked, never executed. */ -declare const epoch: Date; - const queuePublishRuntime: Effect.Effect = producer.publish("message"); const queueBatchRuntime: Effect.Effect = @@ -30,43 +22,7 @@ const queueBatchNeutral: Effect.Effect = produc "message", ]); -declare const policyStore: PolicyCounterStoreShape; - -const requestLimitRuntime: Effect.Effect = - policyStore.checkRequestLimit({ - now: epoch, - projectId: "project", - requestsPerMinute: 1, - }); - -const eventQuotaRuntime: Effect.Effect = - policyStore.checkEventQuota({ - now: epoch, - projectId: "project", - quota: 1, - }); - -// @ts-expect-error Policy counters backed by runtime resources must stay runtime-colored. -const requestLimitNeutral: Effect.Effect = - policyStore.checkRequestLimit({ - now: epoch, - projectId: "project", - requestsPerMinute: 1, - }); - -// @ts-expect-error Policy counters backed by runtime resources must stay runtime-colored. -const eventQuotaNeutral: Effect.Effect = - policyStore.checkEventQuota({ - now: epoch, - projectId: "project", - quota: 1, - }); - void queuePublishRuntime; void queueBatchRuntime; -void requestLimitRuntime; -void eventQuotaRuntime; void queuePublishNeutral; void queueBatchNeutral; -void requestLimitNeutral; -void eventQuotaNeutral; diff --git a/packages/core/test/services/analytics/AnalyticsService.integration.test.ts b/packages/core/test/services/analytics/AnalyticsService.integration.test.ts deleted file mode 100644 index 415362985..000000000 --- a/packages/core/test/services/analytics/AnalyticsService.integration.test.ts +++ /dev/null @@ -1,458 +0,0 @@ -/** - * Integration tests for {@link AnalyticsService}, run against the real backend - * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). - * - * The service is read-only — it writes nothing to MySQL or ClickHouse — so - * these tests need no row cleanup. What they verify instead is the - * *orchestration* layered on top of the (separately unit-tested) pure helpers: - * - * - permission gates: `listRecentEvents` requires `project:all` on the target - * project; `queryAnalyticsInsights` requires `organization:all` on each - * query's organization. Both are exercised via {@link asUnauthorized}, which - * re-authenticates just the guarded call with a no-access session that - * shadows the harness's default full-permission session. - * - the inline `Db` project lookups (org resolution for the recent-events - * tenant setting; the org's project list that seeds filter compilation). - * - composition errors that propagate untouched past the service's - * `catchTags` net (`UnknownInsightError`, `UnsupportedAnalyticsBreakdownError`, - * `InvalidAnalyticsQueryError` for an unsupported granularity, - * `InvalidTimeRangeError` for an inverted custom range). - * - the derived-metric summary contract: a currency insight stamps - * `summary.currency = "USD"`, a rate insight leaves it `undefined`; the - * empty-series branch (no projects match the filter) skips ClickHouse - * entirely and yields a zero summary. - * - the real ClickHouse read path: against the fixture org (which has no - * ingested events) the metric query runs end-to-end and returns an empty - * sparkline rather than throwing. - * - * The pure helpers themselves — filter compilation, time-range presets, the - * insight registry, JSON/date row parsing — are covered by the sibling - * `*.test.ts` unit files and are not re-asserted here. - * - * Typed failures are asserted with `Effect.flip` (project convention), - * narrowing the swapped error with `instanceof` before reading its fields. - */ -import { Clock, DateTime, Effect } from "effect"; -import { describe, expect, test as vitestTest } from "vitest"; - -import { - AnalyticsService, - AnalyticsServiceError, -} from "@voidhash/core/services/analytics/AnalyticsService"; -import { constant } from "@voidhash/lib/lang"; -import { ActionForbiddenError, type UserSession } from "@voidhash/core/domain/auth/Auth"; -import { - InvalidAnalyticsQueryError, - InvalidTimeRangeError, - UnknownInsightError, - UnsupportedAnalyticsBreakdownError, -} from "@voidhash/core/domain/analytics/Analytics"; -import { Db } from "@voidhash/db"; - -import { CoreAuthSession } from "@testing/CoreAuthSession"; -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; -import { CoreTestFixture } from "@testing/CoreTestFixture"; - -const { test } = CoreIntegrationTestHarness.make(); - -const projectId = CoreTestFixture.projectId; -const organizationId = CoreTestFixture.organizationId; - -/** - * A `user`-method session for the fixture principal carrying neither project - * nor organization access — used to assert both permission gates reject. - */ -const EPOCH = DateTime.toDateUtc(DateTime.makeUnsafe(0)); - -const untyped = (value: unknown): any => value; - -const sessionWithoutProjectAccess = (): UserSession => ({ - cookie: null, - method: "user", - name: `${CoreTestFixture.userName} <${CoreTestFixture.userEmail}>`, - organizations: [], - person: null, - projects: [], - user: { - createdAt: EPOCH, - email: CoreTestFixture.userEmail, - emailVerified: true, - id: CoreTestFixture.userId, - image: null, - name: CoreTestFixture.userName, - role: null, - updatedAt: EPOCH, - workosUserId: CoreTestFixture.workosUserId, - }, -}); - -/** - * Run a single call as a caller with no access. Re-authenticates just this - * effect with the no-access session; the inner provision shadows the harness's - * default full-permission session for the wrapped call only. - */ -const asUnauthorized = (effect: Effect.Effect) => - effect.pipe(CoreAuthSession.authenticate(sessionWithoutProjectAccess())); - -describe("AnalyticsService.listRecentEvents", () => { - test( - "returns an events feed against the real ClickHouse for the fixture project", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - // The point is that the query path — org resolution + tenant-scoped - // ClickHouse read — runs end-to-end without throwing and returns a - // well-formed feed. The live ClickHouse may already hold rows for the - // fixture project (seeded by sibling suites / persisted across runs), so - // we assert structure + the limit/hasMore contract rather than an exact - // (and shared-state-dependent) row count. - const limit = 5; - const result = yield* analytics.listRecentEvents({ limit, projectId }); - - expect(Array.isArray(result.events)).toBe(true); - // Never returns more than the requested limit; the +1 lookahead is sliced - // off before mapping (AnalyticsService.ts: rows.slice(0, limit)). - expect(result.events.length).toBeLessThanOrEqual(limit); - // hasMore is `rows.length > limit`, so it can only be true when the page - // is full. - if (result.hasMore) { - expect(result.events.length).toBe(limit); - } - // Any rows that came back must carry the mapped event shape. - for (const event of result.events) { - expect(typeof event.eventId).toBe("string"); - expect(typeof event.eventName).toBe("string"); - expect(event.receivedAt).toBeInstanceOf(Date); - expect(event.processedAt).toBeInstanceOf(Date); - expect(typeof event.context).toBe("object"); - expect(typeof event.properties).toBe("object"); - } - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "resolves the project's organization for the tenant-scoped read", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - const db = yield* Db; - - // Confirm the inline Db lookup the service performs sees the same org we - // expect to be stamped as SQL_organization_id on the ClickHouse query. - const project = yield* db.query.projects.findFirst({ - columns: { organizationId: true }, - where: { id: projectId }, - }); - expect(project?.organizationId).toBe(organizationId); - - // A larger explicit limit must be accepted (clamped to MAX_LIMIT=500 - // internally) and still return cleanly. - const result = yield* analytics.listRecentEvents({ limit: 1000, projectId }); - expect(result.hasMore).toBe(false); - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "forbids callers without project:all and reads nothing", - // No cleanup wrapper: the service writes nothing on any path. - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - const error = yield* Effect.flip(asUnauthorized(analytics.listRecentEvents({ projectId }))); - expect(error).toBeInstanceOf(ActionForbiddenError); - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - vitestTest.todo( - "listRecentEvents: orders by event_ts DESC, honors limit + hasMore, and parses person join / JSON context + properties / dates — requires seeding events_v2 + persons_v1 rows in ClickHouse under the fixture org, which there is no in-process writer seam for here (the analytics writer pipeline runs in a separate worker runtime).", - ); -}); - -describe("AnalyticsService.queryAnalyticsInsights", () => { - test( - "rejects an unknown insight id with UnknownInsightError", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - const error = yield* Effect.flip( - analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - // Untyped on purpose: the service validates the id against the - // registry at runtime; we deliberately pass one outside the - // BuiltInInsightId union to exercise that guard. - insightId: untyped("builtin/does_not_exist"), - key: "k", - timeRange: { preset: "last_7d" }, - }, - ], - }), - ); - expect(error).toBeInstanceOf(UnknownInsightError); - if (error instanceof UnknownInsightError) { - expect(error.insightId).toBe("builtin/does_not_exist"); - } - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "forbids callers without organization:all", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - const error = yield* Effect.flip( - asUnauthorized( - analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - insightId: "builtin/revenue", - key: "k", - timeRange: { preset: "last_7d" }, - }, - ], - }), - ), - ); - expect(error).toBeInstanceOf(ActionForbiddenError); - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "rejects an inverted custom time range with InvalidTimeRangeError", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - const error = yield* Effect.flip( - analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - insightId: "builtin/revenue", - key: "k", - timeRange: { - end: DateTime.toDateUtc(DateTime.makeUnsafe("2024-01-01T00:00:00Z")), - preset: "custom", - start: DateTime.toDateUtc(DateTime.makeUnsafe("2024-06-01T00:00:00Z")), - }, - }, - ], - }), - ); - expect(error).toBeInstanceOf(InvalidTimeRangeError); - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "rejects breakdowns with UnsupportedAnalyticsBreakdownError", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - const error = yield* Effect.flip( - analytics.queryAnalyticsInsights({ - queries: [ - { - breakdowns: [{ field: "product.id" }], - context: { organizationId }, - insightId: "builtin/revenue", - key: "k", - timeRange: { preset: "last_7d" }, - }, - ], - }), - ); - expect(error).toBeInstanceOf(UnsupportedAnalyticsBreakdownError); - if (error instanceof UnsupportedAnalyticsBreakdownError) { - expect(error.field).toBe("product.id"); - } - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "rejects an unsupported granularity for the insight with InvalidAnalyticsQueryError", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - // `builtin/mrr` supports only the non-hourly granularities, so requesting - // `hour` must be rejected. - const error = yield* Effect.flip( - analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - granularity: "hour", - insightId: "builtin/mrr", - key: "k", - timeRange: { preset: "last_7d" }, - }, - ], - }), - ); - expect(error).toBeInstanceOf(InvalidAnalyticsQueryError); - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "returns an empty series and zero summary when the filter matches no projects", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - // Filtering project.id to an id outside the org empties the compiled - // projectIds, so the service short-circuits before any ClickHouse read. - const nowMillis = yield* Clock.currentTimeMillis; - const { results } = yield* analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - filter: { - field: "project.id", - op: "eq", - type: "predicate", - value: `it-nonexistent-project-${nowMillis}`, - }, - insightId: "builtin/revenue", - key: "empty", - timeRange: { preset: "last_7d" }, - }, - ], - }); - - expect(results.length).toBe(1); - const entry = results.find((r) => r.key === "empty"); - expect(entry).toBeDefined(); - expect(entry?.insightId).toBe("builtin/revenue"); - expect(entry?.result.kind).toBe("metric"); - if (entry?.result.kind === "metric") { - expect(entry.result.sparkline).toEqual([]); - expect(entry.result.summary.value).toBe(0); - } - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "stamps USD on a currency insight and leaves currency undefined for a rate insight", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - // Both queries match no projects (empty series) so the summary value is a - // deterministic 0; what differs is the currency stamp, which the service - // derives purely from the insight id (CURRENCY_INSIGHTS vs RATE_INSIGHTS). - const nowMillis = yield* Clock.currentTimeMillis; - const emptyFilter = constant({ - field: "project.id", - op: "eq", - type: "predicate", - value: `it-nonexistent-project-${nowMillis}`, - }); - - const { results } = yield* analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - filter: emptyFilter, - insightId: "builtin/revenue", - key: "currency", - timeRange: { preset: "last_30d" }, - }, - { - context: { organizationId }, - filter: emptyFilter, - insightId: "builtin/churn_rate", - key: "rate", - timeRange: { preset: "last_30d" }, - }, - ], - }); - - const currency = results.find((r) => r.key === "currency"); - const rate = results.find((r) => r.key === "rate"); - - expect(currency?.result.kind).toBe("metric"); - if (currency?.result.kind === "metric") { - expect(currency.result.summary.currency).toBe("USD"); - expect(currency.result.summary.value).toBe(0); - } - - expect(rate?.result.kind).toBe("metric"); - if (rate?.result.kind === "metric") { - expect(rate.result.summary.currency).toBeUndefined(); - expect(rate.result.summary.value).toBe(0); - } - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "runs the real ClickHouse metric read for the org's projects and returns a metric result", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - // No filter → compiled projectIds defaults to the org's project list - // (the fixture project), which is non-empty, so the service issues the - // live tenant-scoped ClickHouse query. The fixture org has no events, so - // the sparkline is empty — but the query must execute, not throw. - const { results } = yield* analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - insightId: "builtin/revenue", - key: "live", - timeRange: { preset: "last_7d" }, - }, - ], - }); - - const entry = results.find((r) => r.key === "live"); - expect(entry).toBeDefined(); - expect(entry?.result.kind).toBe("metric"); - if (entry?.result.kind === "metric") { - expect(Array.isArray(entry.result.sparkline)).toBe(true); - expect(typeof entry.result.summary.value).toBe("number"); - expect(entry.result.summary.currency).toBe("USD"); - } - // The resolved range is surfaced alongside the result and is a real - // ordered interval. - expect(entry?.resolvedTimeRange.start.getTime()).toBeLessThanOrEqual( - entry?.resolvedTimeRange.end.getTime() ?? 0, - ); - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - test( - "resolves multiple insights that share an underlying metric in a single call", - Effect.gen(function* () { - const analytics = yield* AnalyticsService; - - // ARPU and ARPPU both derive from Revenue; queried together they share - // the per-call series-resolver cache. With no events the values are 0, - // but both must resolve to metric results in one call. - const { results } = yield* analytics.queryAnalyticsInsights({ - queries: [ - { - context: { organizationId }, - insightId: "builtin/arpu", - key: "arpu", - timeRange: { preset: "last_7d" }, - }, - { - context: { organizationId }, - insightId: "builtin/arppu", - key: "arppu", - timeRange: { preset: "last_7d" }, - }, - ], - }); - - expect(results.some((r) => r.key === "arpu" && r.result.kind === "metric")).toBe(true); - expect(results.some((r) => r.key === "arppu" && r.result.kind === "metric")).toBe(true); - }).pipe(Effect.provide(AnalyticsService.layer), CoreAuthSession.authenticate()), - ); - - vitestTest.todo( - "queryAnalyticsInsights: returns non-empty sparkline points and a summed/averaged summary for an org with ingested revenue/subscription events — requires seeding events_v2 rows under the fixture org, for which there is no in-process writer seam (analytics ingest runs in a separate worker runtime). The summary aggregation (sum vs avg) and sparkline shape are otherwise covered as pure logic in the sibling unit tests.", - ); -}); - -// Referenced so the wrapped error type is imported and stays in sync with the -// service surface even while its DB/ClickHouse failure paths are deferred above. -void AnalyticsServiceError; diff --git a/packages/core/test/services/analytics/CommunityPostgresAnalytics.integration.test.ts b/packages/core/test/services/analytics/CommunityPostgresAnalytics.integration.test.ts new file mode 100644 index 000000000..4515ea242 --- /dev/null +++ b/packages/core/test/services/analytics/CommunityPostgresAnalytics.integration.test.ts @@ -0,0 +1,183 @@ +import type { CaptureEvent } from "@voidhash/api-contracts/event-capture"; +import type { InternalAnalyticsEvent } from "@voidhash/core/domain/internalAnalytics/InternalAnalyticsEvents"; +import { AnalyticsEventStore } from "@voidhash/core/services/analytics/AnalyticsEventStore"; +import { AnalyticsDispatchService } from "@voidhash/core/services/analyticsIngest/AnalyticsDispatchService"; +import { + type CaptureRequest, + EventCaptureService, +} from "@voidhash/core/services/analyticsIngest/EventCaptureService"; +import { analyticsEvents, apiKeys, Db, eq } from "@voidhash/db"; +import { Clock, DateTime, Effect, Layer } from "effect"; +import { expect } from "vitest"; + +import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; +import { CoreTestFixture } from "@testing/CoreTestFixture"; + +const { test } = CoreIntegrationTestHarness.make(); +let sequence = 0; + +const unique = (prefix: string) => + Effect.map(Clock.currentTimeMillis, (now) => `${prefix}_${now}_${sequence++}`); + +const storeLive = AnalyticsEventStore.layer; +const captureLive = EventCaptureService.layer.pipe(Layer.provide(storeLive)); +const dispatchLive = AnalyticsDispatchService.layer.pipe(Layer.provide(storeLive)); + +const captureEvent = (uuid: string, event: string): typeof CaptureEvent.Type => ({ + uuid, + event, + context: { locale: "en-US" }, + properties: { $app_version: "1.0.0" }, + distinct_id: "device-1", +}); + +const revenueEvent = (eventId: string, now: Date): InternalAnalyticsEvent => ({ + context: {}, + distinctId: "customer-1", + eventId, + eventName: "$purchase.completed", + occurredAt: now, + organizationId: CoreTestFixture.organizationId, + personId: CoreTestFixture.userId, + projectId: CoreTestFixture.projectId, + properties: { + amount: 999, + amountUsd: 999, + currency: "USD", + paymentProviderConfigurationId: "configuration-1", + paymentProviderConfigurationProductId: "configuration-product-1", + providerEnvironment: 1, + providerEventType: "purchase", + providerId: "app-store", + providerSubscriptionId: null, + providerTransactionId: "transaction-1", + providerWebhookNotificationId: null, + source: "sdk", + }, + token: "internal", + transactionId: "transaction-1", +}); + +const withCleanup = ( + eventIds: ReadonlyArray, + apiKeyId: string | undefined, + effect: Effect.Effect, +) => + effect.pipe( + Effect.ensuring( + Effect.gen(function* () { + const db = yield* Db; + for (const eventId of eventIds) { + yield* db.delete(analyticsEvents).where(eq(analyticsEvents.eventId, eventId)); + } + if (apiKeyId) yield* db.delete(apiKeys).where(eq(apiKeys.id, apiKeyId)); + }).pipe(Effect.ignore), + ), + ); + +test( + "OSS capture synchronously stores allow-listed events and rejects all others", + Effect.gen(function* () { + const db = yield* Db; + const now = yield* DateTime.nowAsDate; + const apiKeyId = yield* unique("oss_capture_key"); + const token = `vh_pk_${apiKeyId}`; + const supportedId = yield* unique("oss_supported_event"); + const customId = yield* unique("oss_custom_event"); + const reservedId = yield* unique("oss_reserved_event"); + + yield* db.insert(apiKeys).values({ + end: token.slice(-8), + id: apiKeyId, + isPublic: true, + key: token, + name: "OSS analytics integration key", + prefix: "vh_pk", + projectId: CoreTestFixture.projectId, + }); + + yield* withCleanup( + [supportedId, customId, reservedId], + apiKeyId, + Effect.gen(function* () { + const capture = yield* EventCaptureService; + const request: CaptureRequest = { + events: [ + captureEvent(supportedId, "$app_opened"), + captureEvent(customId, "checkout_started"), + captureEvent(reservedId, "$purchase.completed"), + ], + request: { + headers: {}, + receivedAt: now, + requestId: yield* unique("oss_capture_request"), + sentAt: now, + token, + }, + }; + const result = yield* capture.captureEvents(request); + expect(result).toEqual({ accepted: 1, rejected: 2 }); + + yield* capture.captureEvents(request); + const rows = yield* db + .select() + .from(analyticsEvents) + .where(eq(analyticsEvents.eventId, supportedId)); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + eventName: "$app_opened", + identityMode: "personless", + source: "sdk", + }); + }).pipe(Effect.provide(captureLive)), + ); + }), +); + +test( + "OSS trusted dispatch upserts revenue and skips non-revenue internal events", + Effect.gen(function* () { + const db = yield* Db; + const now = yield* DateTime.nowAsDate; + const eventId = yield* unique("oss_revenue_event"); + const exposureId = yield* unique("oss_exposure_event"); + const revenue = revenueEvent(eventId, now); + const exposure: InternalAnalyticsEvent = { + context: {}, + distinctId: "device-1", + eventId: exposureId, + eventName: "$experiment.exposed", + occurredAt: now, + organizationId: CoreTestFixture.organizationId, + personId: null, + projectId: CoreTestFixture.projectId, + properties: { experimentId: "experiment-1", variantKey: "control" }, + token: "internal", + }; + + yield* withCleanup( + [eventId, exposureId], + undefined, + Effect.gen(function* () { + const dispatch = yield* AnalyticsDispatchService; + yield* dispatch.dispatchTrusted([revenue, revenue, exposure]); + const rows = yield* db + .select() + .from(analyticsEvents) + .where(eq(analyticsEvents.eventId, eventId)); + const exposureRows = yield* db + .select() + .from(analyticsEvents) + .where(eq(analyticsEvents.eventId, exposureId)); + + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + eventName: "$purchase.completed", + identityMode: "full", + source: "revenue", + }); + expect(exposureRows).toHaveLength(0); + }).pipe(Effect.provide(dispatchLive)), + ); + }), +); diff --git a/packages/core/test/services/analytics/clickhouse-accessor.integration.test.ts b/packages/core/test/services/analytics/clickhouse-accessor.integration.test.ts deleted file mode 100644 index 2432abff3..000000000 --- a/packages/core/test/services/analytics/clickhouse-accessor.integration.test.ts +++ /dev/null @@ -1,1763 +0,0 @@ -/** - * Integration tests for the ClickHouse analytics data accessor - * ({@link analyticsAccessor} / the 12 named metric helpers), run against the - * real ClickHouse provisioned once by `test/_testing/globalSetup.ts`. - * - * The accessor is a plain object of pure-SQL query builders — there is no - * service layer to provide. Each helper only requires the - * {@link ClickhouseWebClient} infrastructure service, which the harness already - * supplies (bound to the analytics database's *read-write* user, so a test can - * both seed and assert). - * - * Each test seeds deterministic rows into the real `events_v2` table via - * `ClickhouseWebClient.insertQuery`, runs a metric over a tight, uniquely-namespaced - * time window, and asserts the aggregated `AnalyticsDataPoint[]` the helper - * returns. Conventions: - * - Every seeded row carries a per-test unique `event_id` prefix and a fresh - * `distinct_id`, and queries use a narrow `[start, end]` window so a metric - * only ever sees this test's rows. Assertions stay value/membership-based. - * - Rows are scoped to the shared fixture container - * (`organization_id = it_org`, `project_id = it_project`) so they match the - * accessor's `project_ids` WHERE clause (and the tenant setting, were RLS - * active). - * - {@link withEventCleanup} lightweight-`DELETE`s every seeded row on exit, - * success or failure, via `Effect.ensuring`; the global sweep does not touch - * ClickHouse, so this is the only cleanup. - * - * NOTE on tenant isolation: the harness binds the ClickHouse *read-write* user, - * which is not subject to the readonly role's row policy. The `SQL_organization_id` - * setting the accessor passes is therefore inert here, so org-level RLS - * enforcement cannot be observed in-process — that path is recorded as a - * `test.todo` below. Project scoping (an explicit `project_id IN (...)` WHERE - * clause) IS exercised by every test. - */ -import { constant } from "@voidhash/lib/lang"; -import { Clock, DateTime, Effect, Schema } from "effect"; -import { describe, expect, test as vitestTest } from "vitest"; - -import type { - AnalyticsDataPoint, - CompiledAnalyticsFilter, - TimeGranularity, - TimeRangeParams, -} from "@voidhash/core/domain/analytics/Analytics"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; - -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; -import { CoreTestFixture } from "@testing/CoreTestFixture"; - -import { - type AnalyticsQueryInput, - analyticsAccessor, - getEventFunnelBreakdownCounts, - getEventFunnelCounts, - getEventLifecyclePoints, - getEventPathLinks, - getEventPersonDrilldown, - getEventRetentionCohorts, - getEventStickinessBuckets, - getEventTrendSeries, -} from "../../../src/services/analytics/clickhouse-accessor.ts"; - -const { test } = CoreIntegrationTestHarness.make(); - -// Per-test synthetic project id: each metric seeds and queries `events_v2` over a -// fixed 2021 window. ClickHouse `DELETE` cleanup is eventual and the global -// teardown never sweeps ClickHouse, so a shared project would let a sibling test's -// (or a prior run's) un-deleted rows in the same window bleed into the count. -// `track` (below) sets this to the test's unique namespace before it seeds, so -// every test reads ONLY its own rows regardless of cleanup timing. These reads -// never join MySQL, so the id need not exist there. -let currentProjectId: string = CoreTestFixture.projectId; -const organizationId = CoreTestFixture.organizationId; - -const EVENTS_TABLE = "events_v2"; - -/** Build a fixed instant from an ISO string or epoch millis. */ -const instant = (input: string | number): Date => DateTime.toDateUtc(DateTime.makeUnsafe(input)); - -/** Encodes the `event_properties` column, which ClickHouse stores as JSON text. */ -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); - -/** - * Read once per process so namespaces never collide with a previous run's rows - * (ClickHouse `DELETE` cleanup is eventual). - */ -const RUN_TOKEN = Effect.runSync(Clock.currentTimeMillis); -/** Monotonic counter so namespaces stay unique even within the same millisecond. */ -let seq = 0; -/** A unique token used to prefix every `event_id`/`distinct_id` a test seeds. */ -const uniqueNs = (label: string) => `it-cha-${label}-${RUN_TOKEN}-${seq++}`; - -/** Format a JS `Date` as the `YYYY-MM-DD HH:MM:SS.mmm` string ClickHouse stores. */ -const toEventTs = (date: Date): string => date.toISOString().replace("T", " ").replace("Z", ""); - -/** A single row to seed into `events_v2`. Omitted columns default in JSONEachRow. */ -interface SeedEvent { - readonly eventName: string; - readonly eventTs: Date; - readonly distinctId: string; - readonly personId?: string; - readonly properties?: Record; - /** - * Override the auto-derived `${ns}-${index}` event_id — used to seed two rows - * that share an event_id (a duplicate / redelivery) for the read-side dedup - * tests. Keep it `${ns}-`-prefixed so `deleteSeeded` still cleans it up. - */ - readonly eventId?: string; - /** Latest-wins dedup ordering key (`processed_ts`); defaults to `eventTs`. */ - readonly processedTs?: Date; -} - -/** - * A small fixed time window every metric runs over, plus a base instant inside - * it. Far enough in the past that no real ingest collides with the window, and - * narrow enough that only this test's seeded rows fall inside it. - */ -const startDate = instant("2021-01-01T00:00:00.000Z"); -const endDate = instant("2021-01-31T23:59:59.000Z"); -const baseTs = instant("2021-01-15T12:00:00.000Z"); - -const timeRange = (granularity: TimeGranularity = "day"): TimeRangeParams => ({ - endDate, - granularity, - startDate, -}); - -const filtersFor = (overrides: Partial = {}): CompiledAnalyticsFilter => ({ - projectIds: [currentProjectId], - ...overrides, -}); - -const queryInput = (overrides: Partial = {}): AnalyticsQueryInput => ({ - filters: filtersFor(overrides.filters), - organizationId, - params: overrides.params ?? timeRange(), -}); - -/** Seed rows into the real `events_v2`, namespacing `event_id` with `ns`. */ -const seedEvents = (ns: string, events: ReadonlyArray) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const values = events.map((event, index) => ({ - event_id: event.eventId ?? `${ns}-${index}`, - event_name: event.eventName, - event_ts: toEventTs(event.eventTs), - processed_ts: toEventTs(event.processedTs ?? event.eventTs), - organization_id: organizationId, - project_id: ns, - distinct_id: event.distinctId, - person_id: event.personId ?? null, - event_properties: encodeJson(event.properties ?? {}), - })); - yield* ch.insertQuery({ table: EVENTS_TABLE, values }).pipe(Effect.asVoid); - // `events_v2` is a MergeTree; reads see inserted parts immediately, but be - // explicit so the very next SELECT is fully consistent under the RW user. - yield* ch`SELECT count() FROM ${ch.literal(EVENTS_TABLE)} WHERE event_id LIKE ${ch.param("String", `${ns}-%`)}`; - }); - -/** Lightweight-delete every row a test seeded under `ns`. */ -const deleteSeeded = (ns: string) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - yield* ch - .asCommand( - ch`DELETE FROM ${ch.literal(EVENTS_TABLE)} WHERE event_id LIKE ${ch.param("String", `${ns}-%`)}`, - ) - .pipe(Effect.ignore); - }); - -/** - * Wrap a test body so every event-namespace it seeds is deleted afterward, - * regardless of how the body exits. The body registers each namespace via the - * `track` callback; cleanup reads the collected list lazily at finalization. - */ -const withEventCleanup = ( - body: (track: (ns: string) => string) => Effect.Effect, -): Effect.Effect => { - const namespaces: string[] = []; - const track = (ns: string) => { - namespaces.push(ns); - // Scope this test's seeds and reads to its own project id (see currentProjectId). - currentProjectId = ns; - return ns; - }; - return body(track).pipe( - Effect.ensuring(Effect.forEach(namespaces, deleteSeeded, { discard: true })), - ); -}; - -/** Sum of all data-point values returned by a metric. */ -const totalOf = (points: ReadonlyArray): number => - points.reduce((sum, point) => sum + point.value, 0); - -describe("analyticsAccessor.getRevenue", () => { - test( - "sums amount across purchase/subscription events and converts cents to dollars", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("revenue")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 1000 }, - }, - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 500 }, - }, - // FX-less row: only the raw original-currency `amount` is present (no - // `amount_usd`). USD revenue must NOT fall back to it — summing e.g. - // £2.50 as $2.50 is a silent miscount — so this row contributes 0. - { - eventName: "$subscription.renewed", - eventTs: baseTs, - distinctId, - properties: { amount: 250 }, - }, - // Event outside the metric's event-name set is ignored. - { - eventName: "$subscription.canceled", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 9999 }, - }, - ]); - - const points = yield* analyticsAccessor.getRevenue(queryInput()); - // (1000 + 500) cents -> 15 dollars; the FX-less `amount`-only row is excluded. - expect(totalOf(points)).toBeCloseTo(15, 5); - for (const point of points) { - expect(point.timestamp).toBeInstanceOf(Date); - expect(Number.isNaN(point.timestamp.getTime())).toBe(false); - } - }), - ), - ); - - test( - "returns an empty array (not null) when no events fall in the window", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("revenue-empty")); - // Seed a row OUTSIDE the queried window so nothing matches. - yield* seedEvents(ns, [ - { - eventName: "$purchase.completed", - eventTs: instant("2019-06-01T00:00:00.000Z"), - distinctId: `${ns}-u`, - properties: { amount_usd: 1000 }, - }, - ]); - - const points = yield* analyticsAccessor.getRevenue(queryInput()); - expect(Array.isArray(points)).toBe(true); - expect(points).toEqual([]); - }), - ), - ); - - test( - "honours a non-matching project filter and returns nothing", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("revenue-project")); - yield* seedEvents(ns, [ - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId: `${ns}-u`, - properties: { amount_usd: 4200 }, - }, - ]); - - // Filter on a project the seeded rows do not belong to. - const points = yield* analyticsAccessor.getRevenue( - queryInput({ filters: { projectIds: ["it_project_absent"] } }), - ); - expect(points).toEqual([]); - }), - ), - ); -}); - -describe("analyticsAccessor.getMRR", () => { - test( - "sums subscription revenue but excludes trial subscriptions", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("mrr")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 999, is_trial: false }, - }, - { - eventName: "$subscription.renewed", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 999, is_trial: false }, - }, - // Trial subscription is excluded by the is_trial = 0 guard. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 1500, is_trial: true }, - }, - // Purchase events are not part of the MRR event set. - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 7777 }, - }, - ]); - - const points = yield* analyticsAccessor.getMRR(queryInput()); - // (999 + 999) cents -> 19.98 dollars; trial + purchase excluded. - expect(totalOf(points)).toBeCloseTo(19.98, 5); - }), - ), - ); -}); - -describe("analyticsAccessor.getChurnedRevenue", () => { - test( - "sums amount over subscription.canceled and subscription.expired events", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("churned-rev")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.canceled", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 600 }, - }, - { - eventName: "$subscription.expired", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 400 }, - }, - // Active subscription is not churn. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 9999 }, - }, - ]); - - const points = yield* analyticsAccessor.getChurnedRevenue(queryInput()); - // (600 + 400) cents -> 10 dollars. - expect(totalOf(points)).toBeCloseTo(10, 5); - }), - ), - ); -}); - -describe("analyticsAccessor.getActiveSubscriptions", () => { - test( - "counts distinct non-trial subscription ids", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("active-subs")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-s1`, is_trial: false }, - }, - // Same subscription id seen again -> still one distinct. - { - eventName: "$subscription.renewed", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-s1`, is_trial: false }, - }, - { - eventName: "$subscription.active", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-s2`, is_trial: false }, - }, - // Trial subscription is filtered out by is_trial = 0. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-s3`, is_trial: true }, - }, - ]); - - const points = yield* analyticsAccessor.getActiveSubscriptions(queryInput()); - // s1 + s2 distinct, s3 excluded as trial. - expect(totalOf(points)).toBe(2); - }), - ), - ); -}); - -describe("analyticsAccessor.getActiveTrials", () => { - test( - "counts distinct trial subscription ids only", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("active-trials")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-t1`, is_trial: true }, - }, - { - eventName: "$subscription.active", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-t2`, is_trial: true }, - }, - // Non-trial subscription is excluded by is_trial = 1. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-n1`, is_trial: false }, - }, - ]); - - const points = yield* analyticsAccessor.getActiveTrials(queryInput()); - expect(totalOf(points)).toBe(2); - }), - ), - ); -}); - -describe("analyticsAccessor.getNewSubscriptions", () => { - test( - "counts non-trial subscription.created events", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("new-subs")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-a`, is_trial: false }, - }, - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-b`, is_trial: false }, - }, - // Trial creation excluded by is_trial = 0. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-c`, is_trial: true }, - }, - // Renewal is not a "new" subscription. - { - eventName: "$subscription.renewed", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-a`, is_trial: false }, - }, - ]); - - const points = yield* analyticsAccessor.getNewSubscriptions(queryInput()); - // count() over the two non-trial creations. - expect(totalOf(points)).toBe(2); - }), - ), - ); -}); - -describe("analyticsAccessor.getChurnedSubscriptions", () => { - test( - "counts subscription.canceled and subscription.expired events", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("churned-subs")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.canceled", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-a` }, - }, - { - eventName: "$subscription.expired", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-b` }, - }, - { - eventName: "$subscription.expired", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-c` }, - }, - // Active subscription is not churn. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-d` }, - }, - ]); - - const points = yield* analyticsAccessor.getChurnedSubscriptions(queryInput()); - // count() over the three cancel/expire events. - expect(totalOf(points)).toBe(3); - }), - ), - ); -}); - -describe("analyticsAccessor.getTrials", () => { - test( - "counts trial subscription.created events", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("trials")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-a`, is_trial: true }, - }, - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-b`, is_trial: true }, - }, - // Non-trial creation excluded by is_trial = 1. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-c`, is_trial: false }, - }, - ]); - - const points = yield* analyticsAccessor.getTrials(queryInput()); - expect(totalOf(points)).toBe(2); - }), - ), - ); -}); - -describe("analyticsAccessor.getTrialConversions", () => { - test( - "counts distinct subscription ids flagged converted_from_trial", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("trial-conv")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-a`, converted_from_trial: true }, - }, - // Same subscription id repeated -> still one distinct. - { - eventName: "$subscription.renewed", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-a`, converted_from_trial: true }, - }, - { - eventName: "$subscription.active", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-b`, converted_from_trial: true }, - }, - // Not converted -> excluded by converted_from_trial = 1. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { subscription_id: `${ns}-c`, converted_from_trial: false }, - }, - ]); - - const points = yield* analyticsAccessor.getTrialConversions(queryInput()); - expect(totalOf(points)).toBe(2); - }), - ), - ); -}); - -describe("analyticsAccessor.getPersonCount", () => { - // Regression guard for the empty-string key collapse: with no pending-override - // match the effective-person key must fall through to each event's own - // person_id (the source `nullIf(overrides.col, '')` turns the unmatched join's - // '' back into NULL), so distinct persons stay distinct instead of collapsing - // into a single '' group. - test( - "counts distinct effective persons appearing up to the window end", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("person-count")); - yield* seedEvents(ns, [ - { eventName: "$pageview", eventTs: baseTs, distinctId: `${ns}-d1`, personId: `${ns}-p1` }, - { eventName: "$pageview", eventTs: baseTs, distinctId: `${ns}-d2`, personId: `${ns}-p2` }, - { eventName: "$pageview", eventTs: baseTs, distinctId: `${ns}-d3`, personId: `${ns}-p3` }, - // Same person seen again -> still one distinct person, not double-counted. - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId: `${ns}-d3`, - personId: `${ns}-p3`, - }, - ]); - - const points = yield* analyticsAccessor.getPersonCount(queryInput()); - // Three distinct persons (p1, p2, p3); the repeated p3 event folds in. - expect(totalOf(points)).toBe(3); - }), - ), - ); -}); - -describe("analyticsAccessor.getNewPersons", () => { - test( - "counts only persons first seen inside the window", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("new-persons")); - const beforeWindow = instant("2020-06-01T00:00:00.000Z"); - yield* seedEvents(ns, [ - // Person A's earliest event is BEFORE the window, so min(event_ts) is - // 2020 and the `first_seen >= start` filter excludes it. - { - eventName: "$pageview", - eventTs: beforeWindow, - distinctId: `${ns}-dA`, - personId: `${ns}-pA`, - }, - { eventName: "$pageview", eventTs: baseTs, distinctId: `${ns}-dA`, personId: `${ns}-pA` }, - // Person B is first seen inside the window. - { eventName: "$pageview", eventTs: baseTs, distinctId: `${ns}-dB`, personId: `${ns}-pB` }, - ]); - - const points = yield* analyticsAccessor.getNewPersons(queryInput()); - // Only person B is "new" in [start, end]; person A leaked in from 2020. - expect(totalOf(points)).toBe(1); - }), - ), - ); -}); - -describe("analyticsAccessor.getPayingPersonCount", () => { - test( - "counts distinct persons with a positive-amount purchase or subscription", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("paying-persons")); - yield* seedEvents(ns, [ - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId: `${ns}-dA`, - personId: `${ns}-pA`, - properties: { amount_usd: 1000 }, - }, - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId: `${ns}-dB`, - personId: `${ns}-pB`, - properties: { amount_usd: 500 }, - }, - // Person C's only paying-eligible event has a zero amount, so the - // `amount > 0` guard drops it and C does not count. - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId: `${ns}-dC`, - personId: `${ns}-pC`, - properties: { amount_usd: 0 }, - }, - ]); - - const points = yield* analyticsAccessor.getPayingPersonCount(queryInput()); - // Persons A and B paid; person C's zero-amount row is excluded. - expect(totalOf(points)).toBe(2); - }), - ), - ); -}); - -describe("analyticsAccessor — granularity & period normalisation", () => { - test( - "buckets revenue per day and normalises each period string to a Date", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("granularity-day")); - const distinctId = `${ns}-u`; - const dayA = instant("2021-01-10T08:00:00.000Z"); - const dayB = instant("2021-01-20T20:00:00.000Z"); - yield* seedEvents(ns, [ - { - eventName: "$purchase.completed", - eventTs: dayA, - distinctId, - properties: { amount_usd: 1000 }, - }, - // Same day, different hour -> folds into the same day bucket. - { - eventName: "$purchase.completed", - eventTs: instant("2021-01-10T18:00:00.000Z"), - distinctId, - properties: { amount_usd: 500 }, - }, - { - eventName: "$purchase.completed", - eventTs: dayB, - distinctId, - properties: { amount_usd: 700 }, - }, - ]); - - const points = yield* analyticsAccessor.getRevenue(queryInput()); - // Two distinct day buckets, ordered ascending by period. - expect(points.length).toBe(2); - expect(points[0]?.timestamp.getTime()).toBeLessThan(points[1]?.timestamp.getTime() ?? 0); - // Day buckets normalise to UTC midnight Dates. - const isoDays = points.map((point) => point.timestamp.toISOString().slice(0, 10)); - expect(isoDays).toContain("2021-01-10"); - expect(isoDays).toContain("2021-01-20"); - // Day A bucket = (1000 + 500) cents -> 15 dollars. - const dayAPoint = points.find( - (point) => point.timestamp.toISOString().slice(0, 10) === "2021-01-10", - ); - expect(dayAPoint?.value).toBeCloseTo(15, 5); - }), - ), - ); - - test( - "month granularity collapses events across the window into one bucket", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("granularity-month")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$purchase.completed", - eventTs: instant("2021-01-05T00:00:00.000Z"), - distinctId, - properties: { amount_usd: 1000 }, - }, - { - eventName: "$purchase.completed", - eventTs: instant("2021-01-25T00:00:00.000Z"), - distinctId, - properties: { amount_usd: 1000 }, - }, - ]); - - const points = yield* analyticsAccessor.getRevenue( - queryInput({ params: timeRange("month") }), - ); - // All January events fold into a single month bucket = 20 dollars. - expect(points.length).toBe(1); - expect(points[0]?.timestamp.toISOString().slice(0, 10)).toBe("2021-01-01"); - expect(points[0]?.value).toBeCloseTo(20, 5); - }), - ), - ); -}); - -describe("analyticsAccessor — compiled filter constraints", () => { - test( - "getRevenue applies the product_ids IN filter from CompiledAnalyticsFilter", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("filter-product")); - const distinctId = `${ns}-u`; - const keptProduct = `${ns}-prod-keep`; - yield* seedEvents(ns, [ - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 1000, product_id: keptProduct }, - }, - // Different product -> excluded by the product filter. - { - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 9999, product_id: `${ns}-prod-other` }, - }, - ]); - - const points = yield* analyticsAccessor.getRevenue( - queryInput({ filters: { projectIds: [ns], productIds: [keptProduct] } }), - ); - // Only the kept product's 1000 cents -> 10 dollars. - expect(totalOf(points)).toBeCloseTo(10, 5); - }), - ), - ); - - test( - "getRevenue applies the subscription_status IN filter", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("filter-status")); - const distinctId = `${ns}-u`; - yield* seedEvents(ns, [ - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 1000, subscription_status: 1 }, - }, - // Different status -> excluded. - { - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - properties: { amount_usd: 9999, subscription_status: 2 }, - }, - ]); - - const points = yield* analyticsAccessor.getRevenue( - queryInput({ filters: { projectIds: [ns], subscriptionStatuses: [1] } }), - ); - // Only status 1's 1000 cents -> 10 dollars. - expect(totalOf(points)).toBeCloseTo(10, 5); - }), - ), - ); -}); - -describe("analyticsAccessor — read-side dedup by event_id (latest processed_ts wins)", () => { - test( - "collapses duplicate rows sharing an event_id and keeps the newest processed_ts", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("dedup-revenue")); - const distinctId = `${ns}-u`; - const dupId = `${ns}-dup`; - yield* seedEvents(ns, [ - // Original write of the event. - { - eventId: dupId, - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId, - processedTs: instant("2021-01-15T12:00:00.000Z"), - properties: { amount_usd: 1000 }, - }, - // A redelivery / replay of the SAME event_id with a later processed_ts - // and a corrected amount. The MergeTree keeps both rows; read-side - // latest-wins dedup must keep ONLY this one. - { - eventId: dupId, - eventName: "$purchase.completed", - eventTs: baseTs, - distinctId, - processedTs: instant("2021-01-16T12:00:00.000Z"), - properties: { amount_usd: 1500 }, - }, - ]); - - const points = yield* analyticsAccessor.getRevenue(queryInput()); - // Counted once, and the newer processed_ts (1500 cents) wins -> 15 dollars - // (NOT 25, which is what summing both un-deduped rows would give). - expect(totalOf(points)).toBeCloseTo(15, 5); - }), - ), - ); - - test( - "a count() metric counts a duplicated event_id once, not twice", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("dedup-count")); - const distinctId = `${ns}-u`; - const dupId = `${ns}-c`; - yield* seedEvents(ns, [ - { - eventId: dupId, - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - processedTs: instant("2021-01-15T12:00:00.000Z"), - properties: { subscription_id: `${ns}-s`, is_trial: false }, - }, - // Same event_id written twice (a race / retry) -> count() must see one. - { - eventId: dupId, - eventName: "$subscription.created", - eventTs: baseTs, - distinctId, - processedTs: instant("2021-01-16T12:00:00.000Z"), - properties: { subscription_id: `${ns}-s`, is_trial: false }, - }, - ]); - - const points = yield* analyticsAccessor.getNewSubscriptions(queryInput()); - expect(totalOf(points)).toBe(1); - }), - ), - ); -}); - -describe("getEventTrendSeries", () => { - test( - "aggregates total events and unique users across the full range for number charts", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-trends-total")); - yield* seedEvents(ns, [ - { - eventName: "screen_viewed", - eventTs: instant("2021-01-15T12:00:00.000Z"), - distinctId: `${ns}-one`, - properties: { duration_ms: 10 }, - }, - { - eventName: "screen_viewed", - eventTs: instant("2021-01-16T12:00:00.000Z"), - distinctId: `${ns}-one`, - properties: { duration_ms: 30 }, - }, - { - eventName: "screen_viewed", - eventTs: instant("2021-01-17T12:00:00.000Z"), - distinctId: `${ns}-one`, - }, - ]); - const input = { - aggregateOverRange: true, - eventNames: ["screen_viewed"], - filters: filtersFor(), - organizationId, - params: { endDate, granularity: constant("day"), startDate }, - }; - - const totals = yield* getEventTrendSeries({ ...input, aggregation: "total_events" }); - const uniqueUsers = yield* getEventTrendSeries({ - ...input, - aggregation: "unique_users", - }); - const propertySum = yield* getEventTrendSeries({ - ...input, - aggregation: "property_sum", - mathProperty: "duration_ms", - }); - const propertyAverage = yield* getEventTrendSeries({ - ...input, - aggregation: "property_average", - mathProperty: "duration_ms", - }); - - expect(totals[0]?.points.map((point) => point.value)).toEqual([3]); - expect(uniqueUsers[0]?.points.map((point) => point.value)).toEqual([1]); - expect(propertySum[0]?.points.map((point) => point.value)).toEqual([40]); - expect(propertyAverage[0]?.points.map((point) => point.value)).toEqual([20]); - }), - ), - ); - - test( - "counts unique group actors and applies person-cohort scope before aggregation", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-trends-groups")); - yield* seedEvents(ns, [ - { - eventName: "opened", - eventTs: instant("2021-01-15T12:00:00.000Z"), - distinctId: `${ns}-one-distinct`, - personId: `${ns}-one`, - properties: { account_id: "account-a" }, - }, - { - eventName: "opened", - eventTs: instant("2021-01-16T12:00:00.000Z"), - distinctId: `${ns}-two-distinct`, - personId: `${ns}-two`, - properties: { account_id: "account-a" }, - }, - { - eventName: "opened", - eventTs: instant("2021-01-17T12:00:00.000Z"), - distinctId: `${ns}-three-distinct`, - personId: `${ns}-three`, - properties: { account_id: "account-b" }, - }, - ]); - const input = { - actor: { kind: constant("group"), property: "account_id" }, - aggregateOverRange: true, - aggregation: constant("unique_users"), - eventNames: ["opened"], - filters: filtersFor(), - organizationId, - params: { endDate, granularity: constant("day"), startDate }, - }; - - const allGroups = yield* getEventTrendSeries(input); - const cohortGroups = yield* getEventTrendSeries({ - ...input, - cohortPersonIds: [`${ns}-one`, `${ns}-two`], - }); - - expect(allGroups[0]?.points[0]?.value).toBe(2); - expect(cohortGroups[0]?.points[0]?.value).toBe(1); - }), - ), - ); -}); - -describe("getEventPersonDrilldown", () => { - test( - "returns identified people scoped by cohort membership and group property", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-person-drilldown")); - yield* seedEvents(ns, [ - { - eventName: "screen_viewed", - eventTs: instant("2021-01-15T12:00:00.000Z"), - distinctId: `${ns}-one-distinct`, - personId: `${ns}-one`, - properties: { workspace_id: "mobile" }, - }, - { - eventName: "screen_viewed", - eventTs: instant("2021-01-16T12:00:00.000Z"), - distinctId: `${ns}-one-distinct`, - personId: `${ns}-one`, - properties: { workspace_id: "mobile" }, - }, - { - eventName: "screen_viewed", - eventTs: instant("2021-01-17T12:00:00.000Z"), - distinctId: `${ns}-two-distinct`, - personId: `${ns}-two`, - properties: { workspace_id: "web" }, - }, - { - eventName: "screen_viewed", - eventTs: instant("2021-01-18T12:00:00.000Z"), - distinctId: `${ns}-three-distinct`, - personId: `${ns}-three`, - properties: { workspace_id: "mobile" }, - }, - ]); - - const people = yield* getEventPersonDrilldown({ - cohortPersonIds: [`${ns}-one`, `${ns}-two`], - eventNames: ["screen_viewed"], - group: { property: "workspace_id", value: "mobile" }, - limit: 50, - organizationId, - params: { endDate, startDate }, - projectId: ns, - }); - - expect(people).toEqual([ - { - eventCount: 2, - lastSeenAt: instant("2021-01-16T12:00:00.000Z"), - personId: `${ns}-one`, - }, - ]); - }), - ), - ); -}); - -describe("getEventFunnelCounts", () => { - test( - "distinguishes sequential, strict-adjacency, and any-order funnels", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-funnel-orders")); - const at = (seconds: number) => instant(baseTs.getTime() + seconds * 1000); - yield* seedEvents(ns, [ - { - eventName: "step_a", - eventTs: at(1), - distinctId: `${ns}-sequential`, - properties: { platform: "ios" }, - }, - { eventName: "unrelated", eventTs: at(2), distinctId: `${ns}-sequential` }, - { eventName: "step_b", eventTs: at(3), distinctId: `${ns}-sequential` }, - { - eventName: "step_a", - eventTs: at(4), - distinctId: `${ns}-strict`, - properties: { platform: "android" }, - }, - { eventName: "step_b", eventTs: at(5), distinctId: `${ns}-strict` }, - { eventName: "step_b", eventTs: at(6), distinctId: `${ns}-reverse` }, - { - eventName: "step_a", - eventTs: at(7), - distinctId: `${ns}-reverse`, - properties: { platform: "ios" }, - }, - ]); - const input = { - conversionWindowSeconds: 3_600, - filters: filtersFor(), - organizationId, - params: { endDate, startDate }, - steps: constant([ - { eventNames: constant(["step_a"]), key: "A" }, - { eventNames: constant(["step_b"]), key: "B" }, - ]), - }; - - const sequential = yield* getEventFunnelCounts({ ...input, order: "sequential" }); - const strict = yield* getEventFunnelCounts({ ...input, order: "strict" }); - const any = yield* getEventFunnelCounts({ ...input, order: "any" }); - - expect(sequential).toEqual([3, 2]); - expect(strict).toEqual([3, 1]); - expect(any).toEqual([3, 3]); - - const breakdowns = yield* getEventFunnelBreakdownCounts({ - ...input, - breakdown: { field: "event.properties.platform", limit: 5 }, - breakdownAttributionStep: 1, - order: "sequential", - }); - expect(breakdowns).toEqual([ - { breakdownValue: "android", counts: [1, 1] }, - { breakdownValue: "ios", counts: [2, 1] }, - ]); - }), - ), - ); -}); - -describe("getEventRetentionCohorts", () => { - test( - "distinguishes recurring and first-time cohorts and keeps rolling counts unique", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-retention")); - const day = (offset: number, hour = 12) => - instant( - `2021-01-${String(15 + offset).padStart(2, "0")}T${String(hour).padStart(2, "0")}:00:00.000Z`, - ); - yield* seedEvents(ns, [ - { eventName: "activated", eventTs: day(0), distinctId: `${ns}-one` }, - { eventName: "activated", eventTs: day(1), distinctId: `${ns}-one` }, - { eventName: "opened", eventTs: day(1, 14), distinctId: `${ns}-one` }, - { eventName: "opened", eventTs: day(3), distinctId: `${ns}-one` }, - { eventName: "activated", eventTs: day(0), distinctId: `${ns}-two` }, - { eventName: "opened", eventTs: day(2), distinctId: `${ns}-two` }, - { eventName: "activated", eventTs: day(1), distinctId: `${ns}-three` }, - { eventName: "opened", eventTs: day(3), distinctId: `${ns}-three` }, - ]); - const input = { - cumulative: false, - filters: filtersFor(), - intervals: 4, - organizationId, - params: { endDate, startDate }, - period: constant("day"), - returning: { - aggregation: constant("unique_users"), - eventNames: constant(["opened"]), - key: "returning", - }, - start: { - aggregation: constant("unique_users"), - eventNames: constant(["activated"]), - key: "start", - }, - }; - - const recurring = yield* getEventRetentionCohorts({ - ...input, - retentionType: "recurring", - }); - const firstTime = yield* getEventRetentionCohorts({ - ...input, - retentionType: "first_time", - }); - const rolling = yield* getEventRetentionCohorts({ - ...input, - cumulative: true, - retentionType: "first_time", - }); - - expect(recurring.map((cohort) => [cohort.cohortSize, cohort.counts])).toEqual([ - [2, [0, 1, 1, 1]], - [2, [1, 0, 2, 0]], - ]); - expect(firstTime.map((cohort) => [cohort.cohortSize, cohort.counts])).toEqual([ - [2, [0, 1, 1, 1]], - [1, [0, 0, 1, 0]], - ]); - expect(rolling[0]?.counts).toEqual([0, 2, 2, 1]); - }), - ), - ); -}); - -describe("getEventLifecyclePoints", () => { - test( - "classifies new, returning, resurrecting, and dormant identities by period", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-lifecycle")); - const day = (offset: number, hour = 12) => - instant( - `2021-01-${String(15 + offset).padStart(2, "0")}T${String(hour).padStart(2, "0")}:00:00.000Z`, - ); - yield* seedEvents(ns, [ - { - eventName: "opened", - eventTs: day(0), - distinctId: `${ns}-returning`, - personId: `${ns}-returning-person`, - }, - { - eventName: "opened", - eventTs: day(1), - distinctId: `${ns}-returning`, - personId: `${ns}-returning-person`, - }, - { - eventName: "installed", - eventTs: day(-2), - distinctId: `${ns}-existing`, - personId: `${ns}-existing-person`, - }, - { - eventName: "opened", - eventTs: day(0), - distinctId: `${ns}-existing`, - personId: `${ns}-existing-person`, - }, - { - eventName: "opened", - eventTs: day(0), - distinctId: `${ns}-gap`, - personId: `${ns}-gap-person`, - }, - { - eventName: "opened", - eventTs: day(2), - distinctId: `${ns}-gap`, - personId: `${ns}-gap-person`, - }, - { eventName: "opened", eventTs: day(0), distinctId: `vh:anon:${ns}` }, - ]); - - const points = yield* getEventLifecyclePoints({ - filters: filtersFor(), - granularity: "day", - organizationId, - params: { endDate: day(3, 23), startDate: day(0, 0) }, - series: { - aggregation: "unique_users", - eventNames: ["opened"], - key: "A", - }, - }); - - expect( - points.map((point) => [ - point.timestamp.toISOString().slice(0, 10), - point.status, - point.count, - ]), - ).toEqual([ - ["2021-01-15", "new", 2], - ["2021-01-15", "resurrecting", 1], - ["2021-01-16", "dormant", 2], - ["2021-01-16", "returning", 1], - ["2021-01-17", "dormant", 1], - ["2021-01-17", "resurrecting", 1], - ["2021-01-18", "dormant", 1], - ]); - }), - ), - ); -}); - -describe("getEventPathLinks", () => { - test( - "counts adjacent session transitions, collapses repeats, and maps mobile screen names", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-paths")); - const at = (minutes: number) => instant(baseTs.getTime() + minutes * 60_000); - yield* seedEvents(ns, [ - { eventName: "home", eventTs: at(0), distinctId: `${ns}-one` }, - { eventName: "home", eventTs: at(1), distinctId: `${ns}-one` }, - { eventName: "search", eventTs: at(2), distinctId: `${ns}-one` }, - { eventName: "purchase", eventTs: at(3), distinctId: `${ns}-one` }, - { eventName: "settings", eventTs: at(34), distinctId: `${ns}-one` }, - { eventName: "home", eventTs: at(0), distinctId: `${ns}-two` }, - { eventName: "search", eventTs: at(4), distinctId: `${ns}-two` }, - { eventName: "purchase", eventTs: at(5), distinctId: `${ns}-two` }, - { - eventName: "$screen", - eventTs: at(10), - distinctId: `${ns}-screen`, - properties: { $screen_name: "Welcome" }, - }, - { - eventName: "$screen", - eventTs: at(11), - distinctId: `${ns}-screen`, - properties: { $screen_name: "Paywall" }, - }, - { - eventName: "$screen", - eventTs: at(12), - distinctId: `${ns}-screen`, - properties: { $screen_name: "Done" }, - }, - { eventName: "long_home", eventTs: at(15), distinctId: `${ns}-long` }, - { eventName: "long_middle_a", eventTs: at(16), distinctId: `${ns}-long` }, - { eventName: "long_middle_b", eventTs: at(17), distinctId: `${ns}-long` }, - { eventName: "long_purchase", eventTs: at(18), distinctId: `${ns}-long` }, - ]); - - const eventLinks = yield* getEventPathLinks({ - definition: { - collapseRepeated: true, - eventNames: ["home", "search", "purchase", "settings"], - kind: "paths", - maxDepth: 5, - sessionGapSeconds: 1_800, - timeRange: { preset: "last_30d" }, - }, - filters: filtersFor(), - organizationId, - params: { endDate, startDate }, - }); - const screenLinks = yield* getEventPathLinks({ - definition: { - eventNames: ["$screen"], - kind: "paths", - maxDepth: 5, - pathItem: "screen_name", - timeRange: { preset: "last_30d" }, - }, - filters: filtersFor(), - organizationId, - params: { endDate, startDate }, - }); - const screenLinksWithExclusion = yield* getEventPathLinks({ - definition: { - eventNames: ["$screen"], - excludeEventNames: ["Paywall"], - kind: "paths", - maxDepth: 5, - pathItem: "screen_name", - timeRange: { preset: "last_30d" }, - }, - filters: filtersFor(), - organizationId, - params: { endDate, startDate }, - }); - const collapsedEndpointLinks = yield* getEventPathLinks({ - definition: { - endEventName: "long_purchase", - eventNames: ["long_home", "long_middle_a", "long_middle_b", "long_purchase"], - kind: "paths", - maxDepth: 3, - startEventName: "long_home", - timeRange: { preset: "last_30d" }, - }, - filters: filtersFor(), - organizationId, - params: { endDate, startDate }, - }); - - expect(eventLinks).toEqual([ - { - averageTransitionSeconds: 180, - count: 2, - source: "home", - sourceStep: 1, - target: "search", - targetStep: 2, - }, - { - averageTransitionSeconds: 60, - count: 2, - source: "search", - sourceStep: 2, - target: "purchase", - targetStep: 3, - }, - ]); - expect(screenLinks).toEqual([ - { - averageTransitionSeconds: 60, - count: 1, - source: "Welcome", - sourceStep: 1, - target: "Paywall", - targetStep: 2, - }, - { - averageTransitionSeconds: 60, - count: 1, - source: "Paywall", - sourceStep: 2, - target: "Done", - targetStep: 3, - }, - ]); - expect(screenLinksWithExclusion).toEqual([ - { - averageTransitionSeconds: 120, - count: 1, - source: "Welcome", - sourceStep: 1, - target: "Done", - targetStep: 2, - }, - ]); - expect(collapsedEndpointLinks.map((link) => [link.source, link.target])).toEqual([ - ["long_home", "…"], - ["…", "long_purchase"], - ]); - }), - ), - ); -}); - -describe("getEventStickinessBuckets", () => { - test( - "stitches identities and applies minimum event occurrences inside each interval", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-stickiness")); - const activeAt = (dayOffset: number, hour: number) => - instant( - `2021-01-${String(15 + dayOffset).padStart(2, "0")}T${String(hour).padStart(2, "0")}:00:00.000Z`, - ); - yield* seedEvents(ns, [ - { eventName: "opened", eventTs: activeAt(0, 9), distinctId: `${ns}-three-days` }, - { eventName: "opened", eventTs: activeAt(0, 12), distinctId: `${ns}-three-days` }, - { eventName: "opened", eventTs: activeAt(1, 9), distinctId: `${ns}-three-days` }, - { eventName: "opened", eventTs: activeAt(2, 9), distinctId: `${ns}-three-days` }, - { eventName: "opened", eventTs: activeAt(0, 10), distinctId: `${ns}-two-days` }, - { eventName: "opened", eventTs: activeAt(2, 10), distinctId: `${ns}-two-days` }, - { eventName: "opened", eventTs: activeAt(1, 11), distinctId: `${ns}-one-day` }, - { - eventName: "opened", - eventTs: activeAt(0, 14), - distinctId: `${ns}-stitched-a`, - personId: `${ns}-stitched-person`, - }, - { - eventName: "opened", - eventTs: activeAt(1, 14), - distinctId: `${ns}-stitched-b`, - personId: `${ns}-stitched-person`, - }, - ]); - const input = { - filters: filtersFor(), - interval: constant("day"), - organizationId, - params: { endDate, startDate }, - series: { - aggregation: constant("unique_users"), - eventNames: constant(["opened"]), - key: "A", - }, - }; - - const all = yield* getEventStickinessBuckets({ - ...input, - occurrenceCriteria: { operator: "gte", value: 1 }, - }); - const repeated = yield* getEventStickinessBuckets({ - ...input, - occurrenceCriteria: { operator: "gte", value: 2 }, - }); - - expect(all).toEqual([ - { count: 1, intervals: 1 }, - { count: 2, intervals: 2 }, - { count: 1, intervals: 3 }, - ]); - expect(repeated).toEqual([{ count: 1, intervals: 1 }]); - }), - ), - ); -}); - -describe("custom insight actor scope", () => { - test( - "applies group aggregation and person-cohort membership across every behavioral engine", - withEventCleanup((track) => - Effect.gen(function* () { - const ns = track(uniqueNs("custom-group-cohort-engines")); - const at = (dayOffset: number, minutes: number) => - instant(Date.UTC(2021, 0, 15 + dayOffset, 9, minutes)); - const personOne = `${ns}-person-one`; - const personTwo = `${ns}-person-two`; - const personThree = `${ns}-person-three`; - const group = (accountId: string) => ({ account_id: accountId }); - yield* seedEvents(ns, [ - { - distinctId: `${ns}-one`, - eventName: "funnel_start", - eventTs: at(0, 0), - personId: personOne, - properties: group("account-a"), - }, - { - distinctId: `${ns}-one`, - eventName: "funnel_finish", - eventTs: at(0, 1), - personId: personOne, - properties: group("account-a"), - }, - { - distinctId: `${ns}-three`, - eventName: "funnel_start", - eventTs: at(0, 2), - personId: personThree, - properties: group("account-b"), - }, - { - distinctId: `${ns}-three`, - eventName: "funnel_finish", - eventTs: at(0, 3), - personId: personThree, - properties: group("account-b"), - }, - { - distinctId: `${ns}-one`, - eventName: "retention_start", - eventTs: at(0, 5), - personId: personOne, - properties: group("account-a"), - }, - { - distinctId: `${ns}-two`, - eventName: "retention_return", - eventTs: at(1, 5), - personId: personTwo, - properties: group("account-a"), - }, - { - distinctId: `${ns}-three`, - eventName: "retention_start", - eventTs: at(0, 6), - personId: personThree, - properties: group("account-b"), - }, - { - distinctId: `${ns}-three`, - eventName: "retention_return", - eventTs: at(1, 6), - personId: personThree, - properties: group("account-b"), - }, - { - distinctId: `${ns}-one`, - eventName: "path_home", - eventTs: at(0, 10), - personId: personOne, - properties: group("account-a"), - }, - { - distinctId: `${ns}-two`, - eventName: "path_paywall", - eventTs: at(0, 11), - personId: personTwo, - properties: group("account-a"), - }, - { - distinctId: `${ns}-three`, - eventName: "path_home", - eventTs: at(0, 12), - personId: personThree, - properties: group("account-b"), - }, - { - distinctId: `${ns}-three`, - eventName: "path_paywall", - eventTs: at(0, 13), - personId: personThree, - properties: group("account-b"), - }, - { - distinctId: `${ns}-one`, - eventName: "opened", - eventTs: at(0, 20), - personId: personOne, - properties: group("account-a"), - }, - { - distinctId: `${ns}-two`, - eventName: "opened", - eventTs: at(1, 20), - personId: personTwo, - properties: group("account-a"), - }, - { - distinctId: `${ns}-three`, - eventName: "opened", - eventTs: at(0, 21), - personId: personThree, - properties: group("account-b"), - }, - { - distinctId: `${ns}-three`, - eventName: "opened", - eventTs: at(1, 21), - personId: personThree, - properties: group("account-b"), - }, - ]); - - const actor = { kind: constant("group"), property: "account_id" }; - const cohortPersonIds = [personOne, personTwo]; - const scope = { - actor, - cohortPersonIds, - filters: filtersFor(), - organizationId, - }; - const funnel = yield* getEventFunnelCounts({ - ...scope, - conversionWindowSeconds: 3_600, - order: "sequential", - params: { endDate, startDate }, - steps: [ - { eventNames: ["funnel_start"], key: "A" }, - { eventNames: ["funnel_finish"], key: "B" }, - ], - }); - const retention = yield* getEventRetentionCohorts({ - ...scope, - cumulative: false, - intervals: 3, - params: { endDate: at(2, 59), startDate: at(0, 0) }, - period: "day", - retentionType: "recurring", - returning: { - aggregation: "unique_users", - eventNames: ["retention_return"], - key: "returning", - }, - start: { - aggregation: "unique_users", - eventNames: ["retention_start"], - key: "start", - }, - }); - const paths = yield* getEventPathLinks({ - ...scope, - definition: { - eventNames: ["path_home", "path_paywall"], - kind: "paths", - maxDepth: 3, - timeRange: { preset: "last_30d" }, - }, - params: { endDate, startDate }, - }); - const stickiness = yield* getEventStickinessBuckets({ - ...scope, - interval: "day", - occurrenceCriteria: { operator: "gte", value: 1 }, - params: { endDate, startDate }, - series: { - aggregation: "unique_users", - eventNames: ["opened"], - key: "A", - }, - }); - const lifecycle = yield* getEventLifecyclePoints({ - ...scope, - granularity: "day", - params: { endDate: at(2, 59), startDate: at(0, 0) }, - series: { - aggregation: "unique_users", - eventNames: ["opened"], - key: "A", - }, - }); - - expect(funnel).toEqual([1, 1]); - expect(retention.map((cohort) => [cohort.cohortSize, cohort.counts])).toEqual([ - [1, [0, 1, 0]], - ]); - expect(paths).toEqual([ - { - averageTransitionSeconds: 60, - count: 1, - source: "path_home", - sourceStep: 1, - target: "path_paywall", - targetStep: 2, - }, - ]); - expect(stickiness).toEqual([{ count: 1, intervals: 2 }]); - expect( - lifecycle.map((point) => [ - point.timestamp.toISOString().slice(0, 10), - point.status, - point.count, - ]), - ).toEqual([ - ["2021-01-15", "new", 1], - ["2021-01-16", "returning", 1], - ["2021-01-17", "dormant", 1], - ]); - }), - ), - ); -}); - -// Deferred: the harness binds the ClickHouse *read-write* user, which is not -// subject to the readonly role's row policy, so the `SQL_organization_id` -// per-query setting the accessor passes is inert in-process — a row from -// organization A is still visible when querying as organization B. Verifying -// that the tenant row policy fail-closes for the wrong organization requires -// binding the readonly RLS user (and granting only SELECT under the role), -// which the harness does not expose. Project-level scoping (the explicit -// `project_id IN (...)` WHERE clause) is exercised by the tests above. -vitestTest.todo( - "analyticsAccessor respects organization tenant row policies (needs readonly RLS user, not the harness RW user)", -); diff --git a/packages/core/test/services/analytics/postgres-series-resolver.test.ts b/packages/core/test/services/analytics/postgres-series-resolver.test.ts new file mode 100644 index 000000000..150efeb51 --- /dev/null +++ b/packages/core/test/services/analytics/postgres-series-resolver.test.ts @@ -0,0 +1,51 @@ +import type { StoredAnalyticsEvent } from "@voidhash/core/services/analytics/AnalyticsEventStore"; +import { DateTime } from "effect"; +import { describe, expect, it } from "vitest"; + +import { resolvePostgresAnalyticsSeries } from "../../../src/services/analytics/postgres-series-resolver.ts"; + +const date = (value: string) => DateTime.toDateUtc(DateTime.makeUnsafe(value)); +const timestamp = date("2026-08-01T12:00:00.000Z"); + +const event = ( + sequence: number, + eventName: string, + grossAmountUsd: number, +): StoredAnalyticsEvent => ({ + captureId: `capture-${sequence}`, + context: {}, + distinctId: "customer-1", + eventId: `event-${sequence}`, + eventName, + eventTimestamp: timestamp, + identityMode: "full", + organizationId: "org-1", + personId: "person-1", + previousDistinctId: null, + processedAt: timestamp, + projectId: "project-1", + properties: { grossAmountUsd }, + requestId: `request-${sequence}`, + requestPath: "/internal/analytics", + schemaVersion: 1, + sequence, + sessionId: null, + source: "revenue", + sourceTopic: "revenue.trusted.v1", + token: "internal", +}); + +describe("OSS PostgreSQL revenue analytics", () => { + it("calculates net revenue from signed portable revenue events", () => { + const series = resolvePostgresAnalyticsSeries({ + end: date("2026-08-02T00:00:00.000Z"), + events: [event(1, "$purchase.completed", 1_000), event(2, "$purchase.refunded", -250)], + filters: { projectIds: ["project-1"] }, + granularity: "day", + insightId: "builtin/revenue", + start: date("2026-08-01T00:00:00.000Z"), + }); + + expect(series).toEqual([{ timestamp: date("2026-08-01T00:00:00.000Z"), value: 7.5 }]); + }); +}); diff --git a/packages/core/test/services/analytics/series-resolver.test.ts b/packages/core/test/services/analytics/series-resolver.test.ts deleted file mode 100644 index 4e2cf5a65..000000000 --- a/packages/core/test/services/analytics/series-resolver.test.ts +++ /dev/null @@ -1,421 +0,0 @@ -/** - * Pure unit tests for the analytics series resolver. The resolver's only seam - * is the {@link AnalyticsDataAccessor} (the ClickHouse-backed query layer), - * which is faked here with canned, deterministic series so the derivation graph - * (ARR = MRR x 12, churn-rate, retention, ARPU/ARPPU, *_growth_rate, SLV, trial - * conversion) and the per-call memoisation can be exercised without touching - * ClickHouse. The accessor itself is covered in the integration suite. - * - * The fake also records how many times each metric was invoked so the cache - * behaviour (same key → one underlying call, distinct keys → separate calls) - * can be asserted directly. - */ -import { DateTime, Effect } from "effect"; - -import { describe, expect, it } from "../../../src/testing/effect-vitest.ts"; -import type { - AnalyticsDataPoint, - BuiltInInsightId, - CompiledAnalyticsFilter, - TimeGranularity, -} from "../../../src/domain/analytics/Analytics.ts"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import type { - AnalyticsDataAccessor, - AnalyticsQueryInput, -} from "../../../src/services/analytics/clickhouse-accessor.ts"; -import { buildSeriesResolver } from "../../../src/services/analytics/series-resolver.ts"; - -// ============================================================================= -// Fixtures + a recording accessor fake -// ============================================================================= - -const at = (iso: string): Date => DateTime.toDateUtc(DateTime.makeUnsafe(iso)); - -const TS0 = at("2026-01-01T00:00:00.000Z"); -const TS1 = at("2026-01-02T00:00:00.000Z"); -const TS2 = at("2026-01-03T00:00:00.000Z"); - -/** A fresh data point, one value per period. */ -const point = (timestamp: Date, value: number): AnalyticsDataPoint => ({ timestamp, value }); - -const filter = (overrides: Partial = {}): CompiledAnalyticsFilter => ({ - projectIds: ["it_project"], - ...overrides, -}); - -const timeRange = () => ({ end: TS2, start: TS0 }); - -const granularity: TimeGranularity = "day"; -const organizationId = "it_org"; - -/** - * The resolver's accessor seam, faked. Each metric returns whatever canned - * series the test seeded (default: empty) and bumps a per-metric call counter, - * letting tests assert both the derived values and the cache's effect on the - * number of underlying queries. Unseeded metrics that the resolver never asks - * for simply stay at zero canned data. - * - * The real accessor methods type as `Effect.Effect<…, SqlError, - * ClickhouseWebClient.ClickhouseWebClient>`; `Effect.succeed` (no requirements) - * is assignable to that wider shape, so no ClickHouse service is touched at - * runtime. - */ -interface FakeAccessor { - readonly accessor: AnalyticsDataAccessor; - readonly calls: Record; -} - -type MetricKey = keyof AnalyticsDataAccessor; - -const makeFakeAccessor = ( - series: Partial> = {}, -): FakeAccessor => { - const calls: Record = {}; - const metric = - (key: MetricKey) => - (_input: AnalyticsQueryInput): Effect.Effect => - Effect.sync(() => { - calls[key] = (calls[key] ?? 0) + 1; - // Return a fresh copy so callers cannot mutate the canned fixture. - return (series[key] ?? []).map((p) => ({ ...p })); - }); - - const accessor: AnalyticsDataAccessor = { - getActiveSubscriptions: metric("getActiveSubscriptions"), - getActiveTrials: metric("getActiveTrials"), - getChurnedRevenue: metric("getChurnedRevenue"), - getChurnedSubscriptions: metric("getChurnedSubscriptions"), - getMRR: metric("getMRR"), - getNewPersons: metric("getNewPersons"), - getNewSubscriptions: metric("getNewSubscriptions"), - getPayingPersonCount: metric("getPayingPersonCount"), - getPersonCount: metric("getPersonCount"), - getRevenue: metric("getRevenue"), - getTrialConversions: metric("getTrialConversions"), - getTrials: metric("getTrials"), - }; - - return { accessor, calls }; -}; - -/** - * Stub ClickHouse client to discharge the `ClickhouseWebClient` requirement - * that the resolver's return type carries. The fake accessor never reaches - * ClickHouse, so any access throws (which would signal the fake leaked). - */ -// `ClickhouseWebClient` is a wide `SqlClient` surface that cannot be built here, -// so this narrow helper is the single seam where an untyped value is handed to -// the type system. Every property access dies, so a leak surfaces loudly. -const unusableService = (message: string): any => - new Proxy( - {}, - { - get: () => Effect.runSync(Effect.die(new Error(message))), - }, - ); - -const clickhouseStub: ClickhouseWebClient.ClickhouseWebClient = unusableService( - "ClickhouseWebClient must not be used in this test", -); - -/** Run a resolver effect with the ClickHouse requirement stubbed out. */ -const runSeries = ( - effect: Effect.Effect, -): Effect.Effect => - effect.pipe(Effect.provideService(ClickhouseWebClient.ClickhouseWebClient, clickhouseStub)); - -const get = ( - resolver: ReturnType, - insightId: BuiltInInsightId, -): Effect.Effect => - resolver.getSeries(insightId, filter(), granularity, timeRange(), organizationId); - - -// ============================================================================= -// Rate / growth-rate primitives (exercised through their public callers) -// ============================================================================= - -describe("calculateRate (via churn_rate / retention / trial_conversion_rate)", () => { - it.effect("returns a percentage of numerator over the combined denominator", () => - Effect.gen(function* () { - // churn_rate = churned / (active + churned) * 100 = 1 / (3 + 1) * 100 = 25. - const { accessor } = makeFakeAccessor({ - getActiveSubscriptions: [point(TS0, 3)], - getChurnedSubscriptions: [point(TS0, 1)], - }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/churn_rate")); - expect(result).toEqual([point(TS0, 25)]); - }), - ); - - it.effect("returns 0 when the denominator is 0", () => - Effect.gen(function* () { - // No active and no churned → 0 / 0 → calculateRate guards to 0. - const { accessor } = makeFakeAccessor({ - getActiveSubscriptions: [point(TS0, 0)], - getChurnedSubscriptions: [point(TS0, 0)], - }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/churn_rate")); - expect(result).toEqual([point(TS0, 0)]); - }), - ); -}); - -describe("calculateGrowthRate (via mrr_growth_rate / active_subscribers_growth)", () => { - it.effect("returns 0 for the first period (no previous value, current is 0)", () => - Effect.gen(function* () { - // First point has previous = 0 and current = 0 → 0. - const { accessor } = makeFakeAccessor({ getMRR: [point(TS0, 0), point(TS1, 0)] }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/mrr_growth_rate"), - ); - expect(result[0]).toEqual(point(TS0, 0)); - }), - ); - - it.effect("returns 100 when growing from zero to a positive value", () => - Effect.gen(function* () { - // previous = 0, current > 0 → 100. - const { accessor } = makeFakeAccessor({ getMRR: [point(TS0, 0), point(TS1, 50)] }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/mrr_growth_rate"), - ); - expect(result[1]).toEqual(point(TS1, 100)); - }), - ); - - it.effect("returns ((current - previous) / previous) * 100 between periods", () => - Effect.gen(function* () { - // 200 → 250 = +25%; 250 → 200 = -20%. - const { accessor } = makeFakeAccessor({ - getMRR: [point(TS0, 100), point(TS1, 200), point(TS2, 250)], - }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/mrr_growth_rate"), - ); - expect(result[1]).toEqual(point(TS1, 100)); // 100 → 200 = +100% - expect(result[2]).toEqual(point(TS2, 25)); // 200 → 250 = +25% - }), - ); -}); - -// ============================================================================= -// Per-call cache behaviour -// ============================================================================= - -describe("buildSeriesResolver cache", () => { - it.effect("caches by (insightId, filter, granularity, timeRange, organizationId)", () => - Effect.gen(function* () { - const { accessor, calls } = makeFakeAccessor({ getRevenue: [point(TS0, 10)] }); - const resolver = buildSeriesResolver(accessor); - yield* runSeries(get(resolver, "builtin/revenue")); - yield* runSeries(get(resolver, "builtin/revenue")); - // Identical cache key → only one underlying accessor call. - expect(calls.getRevenue).toBe(1); - }), - ); - - it.effect("reuses the cached result without re-querying for the same key", () => - Effect.gen(function* () { - const { accessor, calls } = makeFakeAccessor({ getMRR: [point(TS0, 7)] }); - const resolver = buildSeriesResolver(accessor); - // ARR and MRR-growth both derive from MRR; with one shared resolver the - // MRR query is issued exactly once across all three reads. - yield* runSeries(get(resolver, "builtin/mrr")); - yield* runSeries(get(resolver, "builtin/arr")); - yield* runSeries(get(resolver, "builtin/mrr_growth_rate")); - expect(calls.getMRR).toBe(1); - }), - ); - - it.effect("queries separately for different cache keys (distinct organization)", () => - Effect.gen(function* () { - const { accessor, calls } = makeFakeAccessor({ getRevenue: [point(TS0, 10)] }); - const resolver = buildSeriesResolver(accessor); - yield* runSeries(get(resolver, "builtin/revenue")); - yield* runSeries( - resolver.getSeries("builtin/revenue", filter(), granularity, timeRange(), "other_org"), - ); - expect(calls.getRevenue).toBe(2); - }), - ); - - it.effect("queries separately for different cache keys (distinct filter)", () => - Effect.gen(function* () { - const { accessor, calls } = makeFakeAccessor({ getRevenue: [point(TS0, 10)] }); - const resolver = buildSeriesResolver(accessor); - yield* runSeries(get(resolver, "builtin/revenue")); - yield* runSeries( - resolver.getSeries( - "builtin/revenue", - filter({ productIds: ["prod_1"] }), - granularity, - timeRange(), - organizationId, - ), - ); - expect(calls.getRevenue).toBe(2); - }), - ); - - it.effect("does not share a cache across separate resolver instances", () => - Effect.gen(function* () { - const { accessor, calls } = makeFakeAccessor({ getRevenue: [point(TS0, 10)] }); - yield* runSeries(get(buildSeriesResolver(accessor), "builtin/revenue")); - yield* runSeries(get(buildSeriesResolver(accessor), "builtin/revenue")); - // Fresh per-call cache each time → two underlying calls. - expect(calls.getRevenue).toBe(2); - }), - ); -}); - -// ============================================================================= -// Derived-metric composition -// ============================================================================= - -describe("derived metrics", () => { - it.effect("ARR maps MRR x 12 over the series", () => - Effect.gen(function* () { - const { accessor } = makeFakeAccessor({ getMRR: [point(TS0, 100), point(TS1, 0)] }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/arr")); - expect(result).toEqual([point(TS0, 1200), point(TS1, 0)]); - }), - ); - - it.effect("churn_rate combines churned and active subscriptions per period", () => - Effect.gen(function* () { - // p0: 1 / (1 + 1) = 50; p1: 0 / (4 + 0) = 0. - const { accessor } = makeFakeAccessor({ - getActiveSubscriptions: [point(TS0, 1), point(TS1, 4)], - getChurnedSubscriptions: [point(TS0, 1), point(TS1, 0)], - }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/churn_rate")); - expect(result).toEqual([point(TS0, 50), point(TS1, 0)]); - }), - ); - - it.effect("retention computes active / (active + churned)", () => - Effect.gen(function* () { - // p0: 3 / (3 + 1) = 75; p1: 0 / (0 + 0) = 0 (guarded). - const { accessor } = makeFakeAccessor({ - getActiveSubscriptions: [point(TS0, 3), point(TS1, 0)], - getChurnedSubscriptions: [point(TS0, 1), point(TS1, 0)], - }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/retention")); - expect(result).toEqual([point(TS0, 75), point(TS1, 0)]); - }), - ); - - it.effect("ARPU divides revenue by person_count with division-by-zero handling", () => - Effect.gen(function* () { - // p0: 100 / 4 = 25; p1: persons = 0 → guarded to 0. - const { accessor } = makeFakeAccessor({ - getRevenue: [point(TS0, 100), point(TS1, 80)], - getPersonCount: [point(TS0, 4), point(TS1, 0)], - }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/arpu")); - expect(result).toEqual([point(TS0, 25), point(TS1, 0)]); - }), - ); - - it.effect("ARPPU divides revenue by paying-person count with division-by-zero handling", () => - Effect.gen(function* () { - // p0: 100 / 2 = 50; p1: paying = 0 → guarded to 0. - const { accessor } = makeFakeAccessor({ - getRevenue: [point(TS0, 100), point(TS1, 60)], - getPayingPersonCount: [point(TS0, 2), point(TS1, 0)], - }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/arppu")); - expect(result).toEqual([point(TS0, 50), point(TS1, 0)]); - }), - ); - - it.effect("MRR growth rate uses the previous-period value", () => - Effect.gen(function* () { - // first period previous = 0 & current 100 → 100; 100 → 150 = +50%. - const { accessor } = makeFakeAccessor({ getMRR: [point(TS0, 100), point(TS1, 150)] }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/mrr_growth_rate"), - ); - expect(result).toEqual([point(TS0, 100), point(TS1, 50)]); - }), - ); - - it.effect("active-subscriber growth uses the previous-period value", () => - Effect.gen(function* () { - // first period previous = 0 & current 10 → 100; 10 → 5 = -50%. - const { accessor } = makeFakeAccessor({ - getActiveSubscriptions: [point(TS0, 10), point(TS1, 5)], - }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/active_subscribers_growth"), - ); - expect(result).toEqual([point(TS0, 100), point(TS1, -50)]); - }), - ); - - it.effect("trial conversion rate divides conversions by trials", () => - Effect.gen(function* () { - // p0: 5 conversions / 10 trials = 50; p1: trials = 0 → guarded to 0. - const { accessor } = makeFakeAccessor({ - getTrials: [point(TS0, 10), point(TS1, 0)], - getTrialConversions: [point(TS0, 5), point(TS1, 3)], - }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/trial_conversion_rate"), - ); - expect(result).toEqual([point(TS0, 50), point(TS1, 0)]); - }), - ); - - it.effect("subscriber lifetime value divides ARPU by the churn fraction", () => - Effect.gen(function* () { - // ARPU = revenue/persons = 100/10 = 10; churn_rate = churned/(active+churned) - // = 5/(15+5) = 25 (percent). SLV = ARPU / (churn% / 100) = 10 / 0.25 = 40. - const { accessor } = makeFakeAccessor({ - getRevenue: [point(TS0, 100)], - getPersonCount: [point(TS0, 10)], - getActiveSubscriptions: [point(TS0, 15)], - getChurnedSubscriptions: [point(TS0, 5)], - }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/subscriber_lifetime_value"), - ); - expect(result).toEqual([point(TS0, 40)]); - }), - ); - - it.effect("subscriber lifetime value is 0 when churn is 0 (no division by zero)", () => - Effect.gen(function* () { - // churn_rate = 0 → SLV guarded to 0. - const { accessor } = makeFakeAccessor({ - getRevenue: [point(TS0, 100)], - getPersonCount: [point(TS0, 10)], - getActiveSubscriptions: [point(TS0, 15)], - getChurnedSubscriptions: [point(TS0, 0)], - }); - const result = yield* runSeries( - get(buildSeriesResolver(accessor), "builtin/subscriber_lifetime_value"), - ); - expect(result).toEqual([point(TS0, 0)]); - }), - ); -}); - -// ============================================================================= -// Primitive pass-through -// ============================================================================= - -describe("primitive metrics", () => { - it.effect("passes a primitive insight straight through the accessor", () => - Effect.gen(function* () { - const { accessor, calls } = makeFakeAccessor({ - getNewPersons: [point(TS0, 3), point(TS1, 7)], - }); - const result = yield* runSeries(get(buildSeriesResolver(accessor), "builtin/new_persons")); - expect(result).toEqual([point(TS0, 3), point(TS1, 7)]); - expect(calls.getNewPersons).toBe(1); - }), - ); -}); diff --git a/packages/core/test/services/analyticsIngest/AnalyticsDispatchService.test.ts b/packages/core/test/services/analyticsIngest/AnalyticsDispatchService.test.ts deleted file mode 100644 index 8582948d4..000000000 --- a/packages/core/test/services/analyticsIngest/AnalyticsDispatchService.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Unit tests for the pure claim-stamping in {@link AnalyticsDispatchService}. - * The enqueue itself is a thin pass-through to {@link CaptureIngress}; the - * load-bearing logic is deriving the SDK identity claim (`Anonymous` vs - * `Stitch`) and stamping `trustClass: "untrusted-sdk"` server-side. - */ -import { describe, expect, it } from "vite-plus/test"; - -import type { CapturedEventV1Type } from "../../../src/domain/analyticsIngest/AnalyticsIngest.ts"; -import { stampSdkIdentityClaim } from "../../../src/services/analyticsIngest/AnalyticsDispatchService.ts"; - -const capturedEvent = (overrides: Partial = {}): CapturedEventV1Type => ({ - schemaVersion: 1, - captureId: "cap_1", - token: "vh_pk_test", - organizationId: "org_1", - projectId: "proj_1", - event: "checkout_started", - distinctId: "dist_1", - eventTimestamp: "2026-03-04T00:00:00.000Z", - receivedAt: "2026-03-04T00:00:00.000Z", - properties: { properties: {}, distinctId: "dist_1" }, - context: {}, - rawPayload: {}, - request: { requestId: "req_1" }, - routing: { - routeClass: "main", - targetTopic: "capture.main.v1", - isHistorical: false, - skipEnrichment: false, - }, - ...overrides, -}); - -describe("stampSdkIdentityClaim", () => { - it("stamps Anonymous for a non-identify event and trustClass untrusted-sdk", () => { - const result = stampSdkIdentityClaim(capturedEvent({ distinctId: "anon_1" })); - expect(result.identityClaim).toEqual({ _tag: "Anonymous", distinctId: "anon_1" }); - expect(result.trustClass).toBe("untrusted-sdk"); - }); - - it("stamps Stitch for an $identify event carrying $previous_distinct_id (nested in properties)", () => { - const result = stampSdkIdentityClaim( - capturedEvent({ - event: "$identify", - distinctId: "user_42", - properties: { properties: { $previous_distinct_id: "anon_7" }, distinctId: "user_42" }, - }), - ); - expect(result.identityClaim).toEqual({ - _tag: "Stitch", - distinctId: "user_42", - previousDistinctId: "anon_7", - }); - expect(result.trustClass).toBe("untrusted-sdk"); - }); - - it("falls back to Anonymous for an $identify event with no usable previous distinct id", () => { - const result = stampSdkIdentityClaim( - capturedEvent({ - event: "$identify", - distinctId: "user_42", - properties: { properties: { $previous_distinct_id: "" }, distinctId: "user_42" }, - }), - ); - expect(result.identityClaim).toEqual({ _tag: "Anonymous", distinctId: "user_42" }); - }); - - it("never sets a Resolved claim from the SDK path (trust cannot be self-asserted)", () => { - const result = stampSdkIdentityClaim(capturedEvent()); - expect(result.identityClaim?._tag).not.toBe("Resolved"); - }); -}); diff --git a/packages/core/test/services/analyticsIngest/AnalyticsIngestDlqService.integration.test.ts b/packages/core/test/services/analyticsIngest/AnalyticsIngestDlqService.integration.test.ts deleted file mode 100644 index add5eac71..000000000 --- a/packages/core/test/services/analyticsIngest/AnalyticsIngestDlqService.integration.test.ts +++ /dev/null @@ -1,468 +0,0 @@ -/** - * Integration tests for {@link AnalyticsIngestDlqService}, run against the real - * backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB). The service writes to / reads from the `analytics_ingest_dlq` - * MySQL table and, for {@link AnalyticsIngestDlqService.requeueFailure}, publishes - * the stored envelope back through the {@link CaptureIngress} port. - * - * Each test drives a public method end-to-end and verifies the *persisted* side - * effects rather than just the return value: - * - the DLQ row written / upserted / marked-replayed in MySQL (read straight - * back via `Db`), - * - for `requeueFailure`, the events handed to a recording {@link CaptureIngress} - * test layer, then the row flipped to `requeued`. - * - * Notes specific to this service: - * - It performs no permission checks and writes no audit-log rows, so there is - * no `ActionForbiddenError` path and no audit cleanup. The default fixture - * session is still provided to satisfy the harness convention. - * - The `analytics_ingest_dlq` table is *not* swept by the global fixture - * teardown, so every test deletes the rows it creates via an - * `Effect.ensuring` finalizer ({@link withDlqCleanup}). - * - `captureId` carries a UNIQUE index; tests that exercise the - * ON-DUPLICATE-KEY upsert reuse a single capture id, all other rows use a - * fresh unique capture id so concurrent/leftover rows never collide. - * - `projectId`/`failureClass` filter assertions use values unique to the test - * (no FK ties this column to the fixture project) and assert by membership, - * never exact counts. - * - `CaptureIngress` is a port boundary outside the harness service set, so the - * test provides its own layer for it. - */ -import { AnalyticsIngestDlqReplayStatus, Db, analyticsIngestDlq, inArray } from "@voidhash/db"; -import { generateId } from "@voidhash/core/utils/generate-id"; -import { Clock, Effect, Layer, Ref } from "effect"; -import { describe, expect } from "vitest"; - -import type { CapturedEventV1Type } from "@voidhash/core/domain/analyticsIngest/AnalyticsIngest"; -import { - type AnalyticsIngestDlqRecordFailureInput, - AnalyticsIngestDlqService, - AnalyticsIngestDlqServiceError, -} from "@voidhash/core/services/analyticsIngest/AnalyticsIngestDlqService"; -import { - CaptureIngress, - type PublishableCaptureEvent, -} from "@voidhash/core/services/analyticsIngest/CaptureIngress"; -import { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; - -import { CoreAuthSession } from "@testing/CoreAuthSession"; -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; - -const { test } = CoreIntegrationTestHarness.make(); - -/** Per-run token plus a monotonic counter so ids/values stay unique across and within runs. */ -const runToken = generateId("test"); -let seq = 0; -const unique = (label: string) => `it-an-dlq-${label}-${runToken}-${seq++}`; - -/** Read a single DLQ row straight from the database, bypassing the service. */ -const findDlqRow = (id: string) => - Effect.gen(function* () { - const db = yield* Db; - return yield* db.query.analyticsIngestDlq.findFirst({ where: { id } }); - }); - -/** - * Delete the given DLQ rows. Each delete is `ignore`d so a missing row never - * turns the finalizer into a failure. This table has no audit rows and no FK - * children, so a single id-based delete suffices. - */ -const cleanupCreatedRows = (ids: ReadonlyArray) => - Effect.gen(function* () { - if (ids.length === 0) return; - const db = yield* Db; - yield* db - .delete(analyticsIngestDlq) - .where(inArray(analyticsIngestDlq.id, [...ids])) - .pipe(Effect.ignore); - }); - -/** - * Wrap a test body so every DLQ row it creates is removed afterward, regardless - * of how the test exits. Pass each created row id to the `track` callback; - * cleanup reads the collected ids lazily at finalization via `Effect.ensuring`, - * so it sees every id tracked while the body ran (including on failure). - */ -const withDlqCleanup = ( - body: (track: (id: string) => void) => Effect.Effect, -): Effect.Effect => { - const createdIds: string[] = []; - return body((id) => { - createdIds.push(id); - }).pipe(Effect.ensuring(cleanupCreatedRows(createdIds))); -}; - -/** A fully-specified `recordFailure` input with sensible, overridable defaults. */ -const recordInput = ( - overrides: Partial = {}, -): AnalyticsIngestDlqRecordFailureInput => ({ - attemptCount: 1, - captureId: unique("cap"), - distinctId: unique("dist"), - failureClass: unique("class"), - failureMessage: "boom", - payloadJson: { hello: "world" }, - projectId: unique("proj"), - routeClass: "main", - sourceSequence: 1, - sourceShard: unique("shard"), - ...overrides, -}); - -/** - * A recording {@link CaptureIngress} layer: every `enqueueBatch` call appends its - * events to a shared `Ref`, which the test reads back to prove `requeueFailure` - * republished the stored envelope and route. Built per layer instance. - */ -/** - * Minimal {@link PlatformRuntime} stub. `requeueFailure` re-publishes through - * `CaptureIngress.enqueueBatch`, whose queue send is colored with the platform - * runtime-phase marker; the recording double below never reads it, so this just - * discharges the type-level requirement in the harness. - */ -const PlatformRuntimeStub = Layer.succeed(PlatformRuntime, PlatformRuntime.of({})); - -const makeRecordingCaptureIngress = (ref: Ref.Ref>) => - Layer.succeed(CaptureIngress, { - enqueueBatch: (events) => Ref.update(ref, (prev) => [...prev, ...events]), - }); - -describe("AnalyticsIngestDlqService.recordFailure", () => { - test( - "inserts a new DLQ row with the supplied fields and a generated id", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const input = recordInput(); - - const id = yield* service.recordFailure(input); - track(id); - expect(id.startsWith("an_ing_dlq_")).toBe(true); - - const row = yield* findDlqRow(id); - expect(row).toBeDefined(); - expect(row?.captureId).toBe(input.captureId); - expect(row?.projectId).toBe(input.projectId); - expect(row?.distinctId).toBe(input.distinctId); - expect(row?.routeClass).toBe(input.routeClass); - expect(row?.failureClass).toBe(input.failureClass); - expect(row?.failureMessage).toBe(input.failureMessage); - expect(row?.attemptCount).toBe(input.attemptCount); - expect(row?.sourceShard).toBe(input.sourceShard); - expect(row?.sourceSequence).toBe(input.sourceSequence); - // payloadJson round-trips through MySQL JSON as the original structure. - expect(row?.payloadJson).toEqual({ hello: "world" }); - // New rows default to Pending and are not yet replayed. - expect(row?.replayStatus).toBe(AnalyticsIngestDlqReplayStatus.Pending); - expect(row?.replayedAt).toBeNull(); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); - - test( - "on a duplicate capture id, updates the existing row (ON DUPLICATE KEY UPDATE) and resets replayStatus to Pending", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const captureId = unique("dup-cap"); - - const firstId = yield* service.recordFailure( - recordInput({ - attemptCount: 1, - captureId, - failureClass: "first_class", - failureMessage: "first failure", - payloadJson: { take: 1 }, - }), - ); - track(firstId); - - // Move the row out of Pending so we can observe the upsert resetting it. - yield* service.markReplayed(firstId); - const afterReplay = yield* findDlqRow(firstId); - expect(afterReplay?.replayStatus).toBe(AnalyticsIngestDlqReplayStatus.Requeued); - - // Second record with the SAME capture id collides on the unique index. - const secondId = yield* service.recordFailure( - recordInput({ - attemptCount: 4, - captureId, - failureClass: "second_class", - failureMessage: "second failure", - payloadJson: { take: 2 }, - }), - ); - // The duplicate may also need cleaning up if MySQL kept the new id row - // (it does not — the upsert mutates the original row), but tracking is - // harmless. - track(secondId); - - // No NEW row exists for the freshly generated id — the upsert mutated - // the original row keyed by capture id. - const secondRow = yield* findDlqRow(secondId); - expect(secondRow).toBeUndefined(); - - const updated = yield* findDlqRow(firstId); - expect(updated).toBeDefined(); - expect(updated?.attemptCount).toBe(4); - expect(updated?.failureClass).toBe("second_class"); - expect(updated?.failureMessage).toBe("second failure"); - expect(updated?.payloadJson).toEqual({ take: 2 }); - // The upsert's `set` resets replayStatus back to Pending. - expect(updated?.replayStatus).toBe(AnalyticsIngestDlqReplayStatus.Pending); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); - - test( - "persists an arbitrary payloadJson shape and the omittable capture/distinct ids", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const payload = { arr: [1, 2, 3], nested: { a: true }, n: 42 }; - - const id = yield* service.recordFailure({ - attemptCount: 0, - failureClass: unique("payload-class"), - failureMessage: "no ids supplied", - payloadJson: payload, - projectId: unique("payload-proj"), - routeClass: "historical", - sourceSequence: 99, - sourceShard: unique("payload-shard"), - }); - track(id); - - const row = yield* findDlqRow(id); - expect(row).toBeDefined(); - expect(row?.captureId).toBeNull(); - expect(row?.distinctId).toBeNull(); - expect(row?.routeClass).toBe("historical"); - expect(row?.payloadJson).toEqual(payload); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); -}); - -describe("AnalyticsIngestDlqService.listFailures", () => { - test( - "returns matching rows ordered by createdAt DESC, respecting the limit", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const projectId = unique("list-proj"); - - const firstId = yield* service.recordFailure(recordInput({ projectId })); - track(firstId); - const secondId = yield* service.recordFailure(recordInput({ projectId })); - track(secondId); - - const rows = yield* service.listFailures({ projectId, limit: 1 }); - // The limit caps the result to the single most recent matching row. - expect(rows.length).toBe(1); - expect(rows.every((row) => row.projectId === projectId)).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); - - test( - "filters by projectId alone", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const projectId = unique("by-proj"); - const otherProjectId = unique("by-proj-other"); - - const mine = yield* service.recordFailure(recordInput({ projectId })); - track(mine); - const other = yield* service.recordFailure(recordInput({ projectId: otherProjectId })); - track(other); - - const rows = yield* service.listFailures({ projectId }); - expect(rows.some((row) => row.id === mine)).toBe(true); - expect(rows.some((row) => row.id === other)).toBe(false); - expect(rows.every((row) => row.projectId === projectId)).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); - - test( - "filters by failureClass alone", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const failureClass = unique("by-class"); - const otherFailureClass = unique("by-class-other"); - - const mine = yield* service.recordFailure(recordInput({ failureClass })); - track(mine); - const other = yield* service.recordFailure( - recordInput({ failureClass: otherFailureClass }), - ); - track(other); - - const rows = yield* service.listFailures({ failureClass }); - expect(rows.some((row) => row.id === mine)).toBe(true); - expect(rows.some((row) => row.id === other)).toBe(false); - expect(rows.every((row) => row.failureClass === failureClass)).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); - - test( - "filters by projectId AND failureClass together", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const projectId = unique("both-proj"); - const failureClass = unique("both-class"); - - // Matches both predicates. - const match = yield* service.recordFailure(recordInput({ failureClass, projectId })); - track(match); - // Same project, different class — must be excluded. - const wrongClass = yield* service.recordFailure( - recordInput({ failureClass: unique("both-class-x"), projectId }), - ); - track(wrongClass); - // Same class, different project — must be excluded. - const wrongProject = yield* service.recordFailure( - recordInput({ failureClass, projectId: unique("both-proj-x") }), - ); - track(wrongProject); - - const rows = yield* service.listFailures({ failureClass, projectId }); - expect(rows.some((row) => row.id === match)).toBe(true); - expect(rows.some((row) => row.id === wrongClass)).toBe(false); - expect(rows.some((row) => row.id === wrongProject)).toBe(false); - expect( - rows.every((row) => row.projectId === projectId && row.failureClass === failureClass), - ).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); - - test( - "clamps an over-large limit to the [1,500] range and still returns matching rows", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const projectId = unique("clamp-proj"); - - const id = yield* service.recordFailure(recordInput({ projectId })); - track(id); - - // limit 9999 is clamped to 500; the call must still succeed and include - // the row we just wrote. - const rows = yield* service.listFailures({ projectId, limit: 9999 }); - expect(rows.length).toBeLessThanOrEqual(500); - expect(rows.some((row) => row.id === id)).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); -}); - -describe("AnalyticsIngestDlqService.markReplayed", () => { - test( - "flips replayStatus to Requeued and stamps replayedAt on the row", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - - const id = yield* service.recordFailure(recordInput()); - track(id); - - const before = yield* findDlqRow(id); - expect(before?.replayStatus).toBe(AnalyticsIngestDlqReplayStatus.Pending); - expect(before?.replayedAt).toBeNull(); - - yield* service.markReplayed(id); - - const after = yield* findDlqRow(id); - expect(after?.replayStatus).toBe(AnalyticsIngestDlqReplayStatus.Requeued); - expect(after?.replayedAt).not.toBeNull(); - }), - ).pipe(Effect.provide(AnalyticsIngestDlqService.layer), CoreAuthSession.authenticate()), - ); -}); - -describe("AnalyticsIngestDlqService.requeueFailure", () => { - test( - "republishes the stored envelope/route via CaptureIngress and marks the row replayed", - withDlqCleanup((track) => - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - // Fresh recording-ingress ref scoped to this single test. - const ref = yield* Ref.make>([]); - - const envelope: CapturedEventV1Type = { - captureId: unique("envelope"), - context: {}, - distinctId: unique("distinct"), - event: "integration_event", - eventTimestamp: "2026-01-01T00:00:00.000Z", - organizationId: unique("organization"), - projectId: unique("project"), - properties: {}, - rawPayload: {}, - receivedAt: "2026-01-01T00:00:00.000Z", - request: { requestId: unique("request") }, - routing: { - isHistorical: false, - routeClass: "overflow", - skipEnrichment: false, - targetTopic: "analytics.integration", - }, - schemaVersion: 1, - token: "integration-token", - }; - const id = yield* service.recordFailure( - recordInput({ payloadJson: envelope, routeClass: "overflow" }), - ); - track(id); - - yield* service.requeueFailure(id).pipe(Effect.provide(makeRecordingCaptureIngress(ref))); - - const enqueued = yield* Ref.get(ref); - expect(enqueued.length).toBe(1); - expect(enqueued[0]?.routeClass).toBe("overflow"); - expect(enqueued[0]?.envelope).toEqual(envelope); - - // The row is flipped to Requeued after a successful publish. - const row = yield* findDlqRow(id); - expect(row?.replayStatus).toBe(AnalyticsIngestDlqReplayStatus.Requeued); - expect(row?.replayedAt).not.toBeNull(); - }), - ).pipe( - Effect.provide(AnalyticsIngestDlqService.layer), - Effect.provide(PlatformRuntimeStub), - CoreAuthSession.authenticate(), - ), - ); - - test( - "fails with AnalyticsIngestDlqServiceError for an unknown row id and publishes nothing", - Effect.gen(function* () { - const service = yield* AnalyticsIngestDlqService; - const ref = yield* Ref.make>([]); - - const nowMillis = yield* Clock.currentTimeMillis; - const missingId = `an_ing_dlq_missing_${nowMillis}`; - const error = yield* Effect.flip( - service.requeueFailure(missingId).pipe(Effect.provide(makeRecordingCaptureIngress(ref))), - ); - expect(error).toBeInstanceOf(AnalyticsIngestDlqServiceError); - if (error instanceof AnalyticsIngestDlqServiceError) { - expect(error.cause).toContain(missingId); - } - - // A not-found requeue never reaches the publish step. - const published = yield* Ref.get(ref); - expect(published.length).toBe(0); - }).pipe( - Effect.provide(AnalyticsIngestDlqService.layer), - Effect.provide(PlatformRuntimeStub), - CoreAuthSession.authenticate(), - ), - ); -}); diff --git a/packages/core/test/services/analyticsIngest/AnalyticsJanitorService.integration.test.ts b/packages/core/test/services/analyticsIngest/AnalyticsJanitorService.integration.test.ts deleted file mode 100644 index dfcbd6651..000000000 --- a/packages/core/test/services/analyticsIngest/AnalyticsJanitorService.integration.test.ts +++ /dev/null @@ -1,407 +0,0 @@ -/** - * Integration tests for {@link AnalyticsJanitorService}, run against the real - * backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * ClickHouse with the analytics database's read-write user, so the test can - * both seed rows and assert their squashed state). - * - * `squash` reconciles pending identity merges in ClickHouse: it reads a backlog - * of `person_identity_pending_overrides_v2` rows older than the safety window, - * materialises a per-run Memory snapshot + Join temp table, bulk-`ALTER TABLE - * UPDATE`s `events_v2.person_id` to the merged person id, deletes the squashed - * backlog rows, then drops the temp tables. Each test drives the operation - * end-to-end and verifies the *persisted* ClickHouse side effects. - * - * Conventions used throughout: - * - `selectBacklog` is GLOBAL (no project filter) and `events_v2` is shared by - * every tenant, so this suite never asserts exact backlog counts. Each test - * instead works under a UNIQUE `project_id` (and unique distinct ids) so its - * seeded rows can't collide with any other tenant's data, and asserts by - * membership: the seeded events were updated and the seeded backlog rows - * were deleted, regardless of whatever else the global squash touched. - * - Every test cleans up after itself in an `Effect.ensuring` finalizer that - * deletes the rows it wrote (events + pending overrides + any leaked temp - * tables) under its own `project_id`, success or failure. The fixture's - * container rows live in MySQL and are irrelevant here. - * - Temp snapshot/join table names are generated dynamically inside the - * service (`makeSnapshotResources`), so the test never asserts their exact - * names — it asserts the *effect* (events updated, backlog deleted) and that - * no `person_identity_pending_override_*` temp tables survive the run. - * - The service takes no `AuthSession`; there is no permission path to cover. - */ -import { Clock, DateTime, Effect } from "effect"; -import { describe, expect, test as vitestTest } from "vitest"; - -import { AnalyticsJanitorService } from "@voidhash/core/services/analyticsIngest/AnalyticsJanitorService"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; - -import { constant } from "@voidhash/lib/lang"; -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; - -const { test } = CoreIntegrationTestHarness.make(); - -const EVENTS_TABLE = constant("events_v2"); -const PENDING_OVERRIDES_TABLE = constant("person_identity_pending_overrides_v2"); - -/** Monotonic counter so ids stay unique even within the same millisecond. */ -let idSeq = 0; -/** A namespace token unique to this run so seeded rows never collide. */ -const uniqueToken = (label: string) => - `it-janitor-${label}-${DateTime.toEpochMillis(DateTime.nowUnsafe())}-${idSeq++}`; - -/** - * Format a `Date` as a ClickHouse `DateTime64(3)` literal (`YYYY-MM-DD - * HH:MM:SS.mmm`, UTC). Mirrors `toClickhouseTimestamp` in the analytics domain - * so seeded `changed_at`/`event_ts` values land in the expected shape. - */ -const chTimestamp = (date: Date): string => { - const pad = (part: number, length = 2) => String(part).padStart(length, "0"); - return [ - `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`, - `${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())}.${pad( - date.getUTCMilliseconds(), - 3, - )}`, - ].join(" "); -}; - -interface SeededProject { - readonly distinctId: string; - readonly organizationId: string; - readonly oldPersonId: string; - readonly mergedPersonId: string; - readonly projectId: string; - readonly version: number; -} - -/** - * Seed one project's worth of ClickHouse state for a squash: - * - an `events_v2` row whose `person_id` is the pre-merge id (or null), - * - a `person_identity_pending_overrides_v2` row mapping - * `(project_id, source_distinct_id) -> mergedPersonId`, timestamped in the - * past so it falls before the safety-window cutoff. - * - * The pending row carries `version > 0` and `is_deleted = 0` so the backlog - * selection (which keeps the latest non-deleted version per - * project_id+source_distinct_id) picks it up. - */ -const seedProject = ( - seed: SeededProject, - options: { readonly eventPersonId: string | null; readonly changedAt: Date }, -) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - yield* ch - .insertQuery({ - table: EVENTS_TABLE, - values: [ - { - event_id: `${seed.projectId}-evt-1`, - capture_id: `${seed.projectId}-cap-1`, - event_name: "integration_event", - event_ts: chTimestamp(options.changedAt), - processed_ts: chTimestamp(options.changedAt), - organization_id: seed.organizationId, - project_id: seed.projectId, - distinct_id: seed.distinctId, - previous_distinct_id: null, - person_id: options.eventPersonId, - identity_mode: "full", - event_properties: "{}", - context: "{}", - route_lane: "main", - skip_enrichment: 0, - source_offset: "0", - source_partition: 0, - source_topic: "capture.v1", - token: seed.projectId, - request_path: "/i/v1/capture", - request_id: `${seed.projectId}-req-1`, - schema_version: 2, - }, - ], - }) - .pipe(Effect.asVoid); - yield* ch - .insertQuery({ - table: PENDING_OVERRIDES_TABLE, - values: [ - { - project_id: seed.projectId, - organization_id: seed.organizationId, - source_distinct_id: seed.distinctId, - target_distinct_id: seed.distinctId, - person_id: seed.mergedPersonId, - is_deleted: 0, - version: seed.version, - changed_at: chTimestamp(options.changedAt), - }, - ], - }) - .pipe(Effect.asVoid); - }); - -/** - * Read the (single, latest) `events_v2.person_id` for a seeded project. - * - * No `FINAL`: the live ClickHouse Cloud `events_v2` uses the `SharedMergeTree` - * storage, which rejects `FINAL` outright ("Storage SharedMergeTree doesn't - * support FINAL"). It's also unnecessary here — each test seeds exactly one - * event row under its unique `project_id`, and the squash rewrites that row in - * place via `ALTER TABLE … UPDATE` (no new version), so the single matching row - * is read back unambiguously. Mirrors how the sibling AnalyticsWriterService - * integration test reads `events_v2` (plain `SELECT … WHERE …`, no `FINAL`). - */ -const findEventPersonId = (projectId: string) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const rows = yield* ch<{ - readonly person_id: string | null; - }>`SELECT person_id FROM ${ch.literal(EVENTS_TABLE)} - WHERE project_id = ${ch.param("String", projectId)} - ORDER BY event_ts DESC - LIMIT 1`; - return rows[0]?.person_id ?? null; - }); - -/** - * Count the non-deleted backlog rows still present for a seeded project. - * - * No `FINAL`: ClickHouse Cloud's `SharedMergeTree` storage rejects it, and it - * isn't needed — each test seeds exactly one pending-override version under its - * unique `project_id`, and a squash physically removes the row via an - * `ALTER TABLE … DELETE` mutation (not a tombstone insert), so an un-squashed - * row reads back as one and a squashed row as zero without any collapse. - */ -const countPendingBacklog = (projectId: string) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const rows = yield* ch<{ - readonly total: string; - }>`SELECT count() AS total FROM ${ch.literal(PENDING_OVERRIDES_TABLE)} - WHERE project_id = ${ch.param("String", projectId)} - AND is_deleted = 0 - AND version > 0`; - return Number(rows[0]?.total ?? "0"); - }); - -/** - * Are any per-run janitor staging table / dictionary objects left behind? - * - * The per-run objects `makeSnapshotResources` creates are named - * `person_identity_pending_override_snapshot_` (a `MergeTree` staging - * table) and `person_identity_pending_override_dict_` (a `Dictionary`, - * which also surfaces in `system.tables` with engine `Dictionary`). ClickHouse - * `LIKE` treats a bare `_` as a single-char wildcard, so the underscores in the - * distinctive `_snapshot_`/`_dict_` infixes are escaped (`\_`) to match them - * literally — otherwise the permanent base table - * `person_identity_pending_overrides_v2` would also match (`override` + `s` for - * the `_` wildcard + `_v2`) and this count would never be zero. - */ -const countLeakedSnapshotTables = () => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const rows = yield* ch<{ readonly total: string }>`SELECT count() AS total FROM system.tables - WHERE database = currentDatabase() - AND ( - name LIKE 'person\\_identity\\_pending\\_override\\_snapshot\\_%' - OR name LIKE 'person\\_identity\\_pending\\_override\\_dict\\_%' - )`; - return Number(rows[0]?.total ?? "0"); - }); - -/** Delete a seeded project's ClickHouse rows (events + backlog), best-effort. */ -const cleanupProject = (projectId: string) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(EVENTS_TABLE)} DELETE WHERE project_id = ${ch.param("String", projectId)} SETTINGS mutations_sync = 1`, - ) - .pipe(Effect.ignore); - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(PENDING_OVERRIDES_TABLE)} DELETE WHERE project_id = ${ch.param("String", projectId)} SETTINGS mutations_sync = 1`, - ) - .pipe(Effect.ignore); - }); - -/** - * Wrap a test body so every project it seeds is removed afterward, regardless - * of how the test exits. Pass each seeded `project_id` to the `track` callback; - * cleanup reads the collected ids lazily at finalization via `Effect.ensuring`. - */ -const withCleanup = ( - body: (track: (projectId: string) => void) => Effect.Effect, -): Effect.Effect => { - const projectIds: string[] = []; - return body((projectId) => { - projectIds.push(projectId); - }).pipe( - Effect.ensuring( - Effect.gen(function* () { - for (const projectId of projectIds) { - yield* cleanupProject(projectId); - } - }), - ), - ); -}; - -describe("AnalyticsJanitorService.squash", () => { - test( - "updates events_v2 person_ids from the backlog, deletes the squashed rows, and reports a positive duration", - withCleanup((track) => - Effect.gen(function* () { - const janitor = yield* AnalyticsJanitorService; - - const projectId = uniqueToken("squash-happy"); - const seed: SeededProject = { - distinctId: `${projectId}-did`, - organizationId: `${projectId}-org`, - mergedPersonId: `${projectId}-person-merged`, - oldPersonId: `${projectId}-person-old`, - projectId, - version: 4, - }; - track(projectId); - - // changed_at is an hour in the past so a zero safety window (cutoff = now) - // includes it in the backlog. - const changedAt = DateTime.toDateUtc( - DateTime.makeUnsafe((yield* Clock.currentTimeMillis) - 60 * 60 * 1000), - ); - yield* seedProject(seed, { changedAt, eventPersonId: seed.oldPersonId }); - - const result = yield* janitor.squash({ batchSize: 10_000, safetyWindowSeconds: 0 }); - - // Our event's person_id is reassigned to the merged id … - expect(yield* findEventPersonId(projectId)).toBe(seed.mergedPersonId); - // … our backlog row is squashed away … - expect(yield* countPendingBacklog(projectId)).toBe(0); - // … no per-run staging table or dictionary survives … - expect(yield* countLeakedSnapshotTables()).toBe(0); - // … and the run reports a valid duration and processed at least our row - // (selectBacklog is global, so other tenants' rows may add to the count). - expect(result.durationMs).toBeGreaterThanOrEqual(0); - expect(result.backlogRowsProcessed).toBeGreaterThanOrEqual(1); - }), - ).pipe(Effect.provide(AnalyticsJanitorService.layer)), - ); - - test( - "assigns the merged person id even when the event's person_id was null (personless → identified)", - withCleanup((track) => - Effect.gen(function* () { - const janitor = yield* AnalyticsJanitorService; - - const projectId = uniqueToken("squash-null-person"); - const seed: SeededProject = { - distinctId: `${projectId}-did`, - organizationId: `${projectId}-org`, - mergedPersonId: `${projectId}-person-merged`, - oldPersonId: `${projectId}-person-old`, - projectId, - version: 5, - }; - track(projectId); - - const changedAt = DateTime.toDateUtc( - DateTime.makeUnsafe((yield* Clock.currentTimeMillis) - 60 * 60 * 1000), - ); - // The event starts personless (person_id null); the squash's - // `person_id IS NULL` predicate still matches it for reassignment. - yield* seedProject(seed, { changedAt, eventPersonId: null }); - - yield* janitor.squash({ batchSize: 10_000, safetyWindowSeconds: 0 }); - - expect(yield* findEventPersonId(projectId)).toBe(seed.mergedPersonId); - expect(yield* countPendingBacklog(projectId)).toBe(0); - }), - ).pipe(Effect.provide(AnalyticsJanitorService.layer)), - ); - - test( - "skips backlog rows newer than the safety window and leaves the matching events untouched", - withCleanup((track) => - Effect.gen(function* () { - const janitor = yield* AnalyticsJanitorService; - - const projectId = uniqueToken("safety-window"); - const seed: SeededProject = { - distinctId: `${projectId}-did`, - organizationId: `${projectId}-org`, - mergedPersonId: `${projectId}-person-merged`, - oldPersonId: `${projectId}-person-old`, - projectId, - version: 3, - }; - track(projectId); - - // The backlog row is recent (just now). A large safety window pushes the - // cutoff far into the past, so this row is NEWER than the cutoff and the - // backlog selection (`changed_at < cutoff`) must exclude it. - yield* seedProject(seed, { - changedAt: yield* DateTime.nowAsDate, - eventPersonId: seed.oldPersonId, - }); - - yield* janitor.squash({ batchSize: 10_000, safetyWindowSeconds: 86_400 }); - - // Our event keeps its pre-merge person id … - const eventPersonId = yield* findEventPersonId(projectId); - expect(eventPersonId).toBe(seed.oldPersonId); - - // … and our backlog row is untouched (not squashed away). - expect(yield* countPendingBacklog(projectId)).toBe(1); - }), - ).pipe(Effect.provide(AnalyticsJanitorService.layer)), - ); - - test( - "floors a non-positive batchSize to zero so the run touches no events but still returns a valid result", - withCleanup((track) => - Effect.gen(function* () { - const janitor = yield* AnalyticsJanitorService; - - const projectId = uniqueToken("zero-batch"); - const seed: SeededProject = { - distinctId: `${projectId}-did`, - organizationId: `${projectId}-org`, - mergedPersonId: `${projectId}-person-merged`, - oldPersonId: `${projectId}-person-old`, - projectId, - version: 2, - }; - track(projectId); - - const changedAt = DateTime.toDateUtc( - DateTime.makeUnsafe((yield* Clock.currentTimeMillis) - 60 * 60 * 1000), - ); - yield* seedProject(seed, { changedAt, eventPersonId: seed.oldPersonId }); - - // batchSize=0 floors the `LIMIT {row_limit}` to 0 → no rows selected, - // so the early-return path runs and nothing is mutated. - const result = yield* janitor.squash({ batchSize: 0, safetyWindowSeconds: 0 }); - - expect(result.backlogRowsProcessed).toBe(0); - expect(typeof result.cutoffIso).toBe("string"); - expect(result.durationMs).toBeGreaterThanOrEqual(0); - - // The empty-backlog branch never creates temp tables, so none leak. - expect(yield* countLeakedSnapshotTables()).toBe(0); - - // Our event and backlog row are both untouched. - expect(yield* findEventPersonId(projectId)).toBe(seed.oldPersonId); - expect(yield* countPendingBacklog(projectId)).toBe(1); - }), - ).pipe(Effect.provide(AnalyticsJanitorService.layer)), - ); - - vitestTest.todo( - "wraps an underlying ClickHouse failure as AnalyticsJanitorServiceError carrying the operation context — deferred: no schema-valid input deterministically forces a ClickhouseError on the live ClickHouse Cloud instance. Both attempted triggers were tolerated rather than rejected: a far-future cutoff returns an empty backlog (squash succeeds, backlogRowsProcessed:0), and a >UInt32 batchSize (5_000_000_000 bound into LIMIT {row_limit:UInt32}) is accepted/clamped instead of parse-failing. Forcing the catchTags→AnalyticsJanitorServiceError wrap would need a fault-injecting Clickhouse double, which the integration tier forbids; cover it in the unit tier with a stubbed Clickhouse layer whose `query`/`command` fails.", - ); - - vitestTest.todo( - "logs (but does not re-throw) a temp-table DROP failure during cleanup — deferred: cleanupSnapshotResources swallows DROP errors via Effect.logError, and there is no in-process seam to force a DROP to fail without a fault-injecting ClickHouse double, which the integration tier forbids. Verifying the swallow would need a stubbed Clickhouse layer (unit tier) where `command` selectively fails on the DROP statements.", - ); -}); diff --git a/packages/core/test/services/analyticsIngest/AnalyticsWriterService.integration.test.ts b/packages/core/test/services/analyticsIngest/AnalyticsWriterService.integration.test.ts deleted file mode 100644 index 88dbd7f33..000000000 --- a/packages/core/test/services/analyticsIngest/AnalyticsWriterService.integration.test.ts +++ /dev/null @@ -1,528 +0,0 @@ -/** - * Integration tests for {@link AnalyticsWriterService}, run against the real - * backend stack provisioned once by `test/_testing/globalSetup.ts`. Both - * storage backends are live: MySQL (the `project → organization` lookup) and - * ClickHouse (the five fan-out tables). The harness binds ClickHouse's - * read-write user (see `stacks/backend.ts` `makeTestConnections`), which has - * `GRANT ALL` and reads cross-tenant rows with no row policy — so this test can - * both insert and read its own rows back directly via - * {@link ClickhouseWebClient}, without a `SQL_organization_id` setting. - * - * Each test drives `writeMessages` end-to-end and verifies the *persisted* side - * effects, not just the returned counts: - * - processed-event rows land in `events_v2` (read back by `event_id`), - * - person / identity / override / pending-override rows land in their tables, - * - the `organization_id` written onto person/identity rows is the one - * resolved from MySQL `project` (not the upstream-stamped value), - * - the three dedup paths (within-batch revenue, ClickHouse-existing event, - * ClickHouse-existing revenue within the safety window) actually drop rows - * before insert, so `insertedRowCount` reflects real inserts. - * - * Conventions: - * - `writeMessages` carries no `AuthSession` permission guard, so there is no - * forbidden-path case; the harness skeleton's `CoreAuthSession.authenticate()` - * is still applied for uniformity. - * - Every test namespaces its rows with a unique `event_id` / `distinct_id` / - * `person_id` (via {@link uniqueId}) and asserts by membership, never by a - * table-wide count, so a leftover row from a crashed run can't collide. - * - ClickHouse `MergeTree` has no cascading delete, so {@link withClickhouseCleanup} - * issues a best-effort `ALTER TABLE … DELETE WHERE event_id|distinct_id IN (…)` - * mutation per table on exit (success or failure) via `Effect.ensuring`. The - * global MySQL sweep does not touch ClickHouse, so this self-cleanup is the - * only thing that reclaims these rows. - * - Typed failures are asserted with `Effect.flip` + `instanceof`, paired with - * a state assertion on the failure path (project convention). - */ -import { Clock, DateTime, Effect } from "effect"; -import { describe, expect, test as vitestTest } from "vitest"; - -import { AnalyticsWriterService } from "@voidhash/core/services/analyticsIngest/AnalyticsWriterService"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import type { AnalyticsWriterMessageType } from "@voidhash/core/domain/analyticsIngest/AnalyticsIngest"; -import { REVENUE_TRUSTED_SOURCE_TOPIC } from "@voidhash/core/domain/internalAnalytics/InternalAnalyticsEvents"; - -import { CoreAuthSession } from "@testing/CoreAuthSession"; -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; -import { CoreTestFixture } from "@testing/CoreTestFixture"; - -const { test } = CoreIntegrationTestHarness.make(); - -const projectId = CoreTestFixture.projectId; -const organizationId = CoreTestFixture.organizationId; - -const EVENTS_TABLE = "events_v2"; -const PERSONS_TABLE = "persons_v1"; -const PERSON_IDENTITY_TABLE = "person_identity_v1"; -const PERSON_IDENTITY_OVERRIDES_TABLE = "person_identity_overrides_v1"; -const PERSON_IDENTITY_PENDING_OVERRIDES_TABLE = "person_identity_pending_overrides_v2"; - -/** Monotonic counter so ids stay unique even within the same millisecond. */ -let idSeq = 0; -const uniqueId = (label: string) => - Effect.map(Clock.currentTimeMillis, (now) => `it-aw-${label}-${now}-${idSeq++}`); - -/** A reservation-grade revenue event name (drives the revenue dedup path). */ -const REVENUE_EVENT_NAME = "$purchase.completed"; - -/** - * Build a `processed` writer message. The default `event` is a customer - * (non-revenue) event; pass `{ event, sourceTopic }` overrides to make it a - * trusted revenue event that the dedup helpers recognize. `eventTimestamp` - * controls the row's `event_ts`, which the revenue safety-window cutoff reads. - */ -const processedMessage = (overrides: { - readonly event?: string; - readonly eventId: string; - readonly distinctId?: string; - readonly eventTimestamp?: string; - readonly sourceTopic?: string; -}): Effect.Effect => - Effect.gen(function* () { - const distinctId = overrides.distinctId ?? (yield* uniqueId("distinct")); - const eventTimestamp = overrides.eventTimestamp ?? (yield* DateTime.nowAsDate).toISOString(); - const personId = yield* uniqueId("person"); - const token = yield* uniqueId("token"); - return { - kind: "processed", - messageId: overrides.eventId, - value: { - captureId: `cap-${overrides.eventId}`, - context: {}, - distinctId, - event: overrides.event ?? "checkout_started", - eventTimestamp, - groups: [], - identity: { distinctId, mode: "full", personId }, - organizationId, - processedAt: eventTimestamp, - processedEventId: overrides.eventId, - projectId, - properties: { plan: "pro" }, - request: { path: "/i/v1/capture", requestId: `req-${overrides.eventId}` }, - routing: { - lane: "main", - skipEnrichment: false, - sourceOffset: overrides.eventId, - sourcePartition: 0, - sourceTopic: overrides.sourceTopic ?? "capture.v1", - }, - schemaVersion: 2, - token, - }, - }; - }); - -/** - * Build a `person` writer message under the fixture project. The - * `primaryDistinctId` is taken from the caller so the test can `trackDistinct` - * it: the `persons_v1` cleanup keys on `primary_distinct_id`, so the row would - * otherwise leak (the global MySQL sweep never touches ClickHouse). - */ -const personMessage = (overrides: { - readonly personId: string; - readonly primaryDistinctId: string; -}): Effect.Effect => - Effect.map(DateTime.nowAsDate, (now) => ({ - kind: "person", - messageId: overrides.personId, - value: { - changedAt: now.toISOString(), - isArchived: false, - name: "Integration Person", - personId: overrides.personId, - primaryDistinctId: overrides.primaryDistinctId, - projectId, - schemaVersion: 1, - traits: { tier: "gold" }, - version: 1, - }, - })); - -/** - * Build a `person-distinct-id` writer message. A non-empty `previousDistinctId` - * with `version > 0` makes the plan also emit override + pending-override rows. - */ -const personIdentityMessage = (overrides: { - readonly personId: string; - readonly distinctId: string; - readonly previousDistinctId?: string; - readonly version?: number; -}): Effect.Effect => - Effect.map(DateTime.nowAsDate, (now) => ({ - kind: "person-distinct-id", - messageId: overrides.personId, - value: { - changedAt: now.toISOString(), - distinctId: overrides.distinctId, - isDeleted: false, - personId: overrides.personId, - previousDistinctId: overrides.previousDistinctId, - projectId, - schemaVersion: 1, - version: overrides.version ?? 1, - }, - })); - -/** Read back the processed-event rows for the given ids straight from ClickHouse. */ -const findEventRows = (eventIds: ReadonlyArray) => - Effect.gen(function* () { - if (eventIds.length === 0) return []; - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - return yield* ch<{ - event_id: string; - event_name: string; - organization_id: string; - project_id: string; - distinct_id: string; - source_topic: string; - }>`SELECT event_id, event_name, organization_id, project_id, distinct_id, source_topic - FROM ${ch.literal(EVENTS_TABLE)} WHERE event_id IN ${ch.param("Array(String)", [...eventIds])}`; - }); - -/** Read back the person rows for the given person ids. */ -const findPersonRows = (personIds: ReadonlyArray) => - Effect.gen(function* () { - if (personIds.length === 0) return []; - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - return yield* ch<{ - person_id: string; - organization_id: string; - project_id: string; - }>`SELECT person_id, organization_id, project_id - FROM ${ch.literal(PERSONS_TABLE)} WHERE person_id IN ${ch.param("Array(String)", [...personIds])}`; - }); - -/** The distinct-id column each identity table keys its rows on. */ -const identityColumn = (table: string): string => { - if (table === PERSON_IDENTITY_PENDING_OVERRIDES_TABLE) { - return "target_distinct_id"; - } - return "distinct_id"; -}; - -/** Read back rows from one of the three identity tables by `distinct_id`. */ -const findIdentityRows = (table: string, distinctIds: ReadonlyArray) => - Effect.gen(function* () { - if (distinctIds.length === 0) return []; - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const column = identityColumn(table); - return yield* ch<{ - person_id: string; - organization_id: string; - project_id: string; - }>`SELECT person_id, organization_id, project_id - FROM ${ch.literal(table)} WHERE ${ch.literal(column)} IN ${ch.param("Array(String)", [...distinctIds])}`; - }); - -/** - * Best-effort ClickHouse reclamation. MergeTree mutations are asynchronous, so - * each `ALTER TABLE … DELETE` is fire-and-forget and `ignore`d — a failed or - * slow mutation must never turn the finalizer into a test failure. Events are - * cleared by `event_id`; the four person/identity tables by `distinct_id` - * (`target_distinct_id` for the pending-overrides table). - */ -const cleanupClickhouse = (created: { - readonly eventIds: ReadonlyArray; - readonly distinctIds: ReadonlyArray; -}) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const eventIds = [...created.eventIds]; - const distinctIds = [...created.distinctIds]; - - if (eventIds.length > 0) { - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(EVENTS_TABLE)} DELETE WHERE event_id IN ${ch.param("Array(String)", eventIds)}`, - ) - .pipe(Effect.ignore); - } - if (distinctIds.length > 0) { - for (const table of [PERSONS_TABLE]) { - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(table)} DELETE WHERE primary_distinct_id IN ${ch.param("Array(String)", distinctIds)}`, - ) - .pipe(Effect.ignore); - } - for (const table of [PERSON_IDENTITY_TABLE, PERSON_IDENTITY_OVERRIDES_TABLE]) { - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(table)} DELETE WHERE distinct_id IN ${ch.param("Array(String)", distinctIds)}`, - ) - .pipe(Effect.ignore); - } - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(PERSON_IDENTITY_PENDING_OVERRIDES_TABLE)} DELETE WHERE target_distinct_id IN ${ch.param("Array(String)", distinctIds)}`, - ) - .pipe(Effect.ignore); - } - }); - -/** - * Wrap a test body so every ClickHouse row it writes is reclaimed afterward, - * regardless of how the test exits. Pass each written `event_id` to `trackEvent` - * and each `distinct_id` (events + identity tables key on it) to `trackDistinct`; - * cleanup reads the collected ids lazily at finalization via `Effect.ensuring`. - */ -const withClickhouseCleanup = ( - body: (track: { - readonly trackEvent: (id: string) => void; - readonly trackDistinct: (id: string) => void; - }) => Effect.Effect, -): Effect.Effect => { - const eventIds: string[] = []; - const distinctIds: string[] = []; - return body({ - trackDistinct: (id) => { - distinctIds.push(id); - }, - trackEvent: (id) => { - eventIds.push(id); - }, - }).pipe(Effect.ensuring(cleanupClickhouse({ distinctIds, eventIds }))); -}; - -describe("AnalyticsWriterService.writeMessages", () => { - test( - "fans the three message kinds out into all five ClickHouse tables in one batch", - withClickhouseCleanup(({ trackDistinct, trackEvent }) => - Effect.gen(function* () { - const writer = yield* AnalyticsWriterService; - - const eventId = yield* uniqueId("fanout-event"); - const eventDistinct = yield* uniqueId("fanout-event-distinct"); - const personId = yield* uniqueId("fanout-person"); - const personPrimaryDistinct = yield* uniqueId("fanout-person-primary"); - const identityDistinct = yield* uniqueId("fanout-identity-distinct"); - const identityPrevious = yield* uniqueId("fanout-identity-prev"); - trackEvent(eventId); - trackDistinct(eventDistinct); - trackDistinct(personPrimaryDistinct); - trackDistinct(identityDistinct); - - const messages: ReadonlyArray = [ - yield* processedMessage({ distinctId: eventDistinct, eventId }), - yield* personMessage({ personId, primaryDistinctId: personPrimaryDistinct }), - // version > 0 + a previous distinct id → also emits override + - // pending-override rows, exercising all five tables at once. - yield* personIdentityMessage({ - distinctId: identityDistinct, - personId, - previousDistinctId: identityPrevious, - version: 3, - }), - ]; - - const result = yield* writer.writeMessages(messages); - // 1 processed + 1 person + 1 identity + 1 override + 1 pending = 5 rows. - expect(result.insertedRowCount).toBe(5); - expect(result.messageCount).toBe(3); - - const eventRows = yield* findEventRows([eventId]); - expect(eventRows.some((row) => row.event_id === eventId)).toBe(true); - - const personRows = yield* findPersonRows([personId]); - expect(personRows.some((row) => row.person_id === personId)).toBe(true); - - const identityRows = yield* findIdentityRows(PERSON_IDENTITY_TABLE, [identityDistinct]); - expect(identityRows.some((row) => row.person_id === personId)).toBe(true); - - const overrideRows = yield* findIdentityRows(PERSON_IDENTITY_OVERRIDES_TABLE, [ - identityDistinct, - ]); - expect(overrideRows.some((row) => row.person_id === personId)).toBe(true); - - const pendingRows = yield* findIdentityRows(PERSON_IDENTITY_PENDING_OVERRIDES_TABLE, [ - identityDistinct, - ]); - expect(pendingRows.some((row) => row.person_id === personId)).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsWriterService.layer), CoreAuthSession.authenticate()), - ); - - test( - "resolves project → organization from MySQL and stamps it on person/identity rows", - withClickhouseCleanup(({ trackDistinct }) => - Effect.gen(function* () { - const writer = yield* AnalyticsWriterService; - - const personId = yield* uniqueId("org-person"); - const personPrimaryDistinct = yield* uniqueId("org-person-primary"); - const identityDistinct = yield* uniqueId("org-identity-distinct"); - trackDistinct(personPrimaryDistinct); - trackDistinct(identityDistinct); - - // Person/identity messages carry only projectId; the writer looks up the - // organization in MySQL. The fixture seeds it_project under it_org. - yield* writer.writeMessages([ - yield* personMessage({ personId, primaryDistinctId: personPrimaryDistinct }), - yield* personIdentityMessage({ distinctId: identityDistinct, personId }), - ]); - - const personRows = yield* findPersonRows([personId]); - const personRow = personRows.find((row) => row.person_id === personId); - expect(personRow).toBeDefined(); - expect(personRow?.organization_id).toBe(organizationId); - expect(personRow?.project_id).toBe(projectId); - - const identityRows = yield* findIdentityRows(PERSON_IDENTITY_TABLE, [identityDistinct]); - const identityRow = identityRows.find((row) => row.person_id === personId); - expect(identityRow).toBeDefined(); - expect(identityRow?.organization_id).toBe(organizationId); - }), - ).pipe(Effect.provide(AnalyticsWriterService.layer), CoreAuthSession.authenticate()), - ); - - test( - "deduplicates revenue events sharing an event_id within the batch (only one row inserted)", - withClickhouseCleanup(({ trackEvent }) => - Effect.gen(function* () { - const writer = yield* AnalyticsWriterService; - - const sharedEventId = yield* uniqueId("batch-dup-event"); - trackEvent(sharedEventId); - - // Two trusted revenue messages with the SAME event_id: within-batch dedup - // keeps the first and skips the second. The two non-revenue events have - // distinct ids and pass through untouched. - const passEventA = yield* uniqueId("batch-pass-a"); - const passEventB = yield* uniqueId("batch-pass-b"); - trackEvent(passEventA); - trackEvent(passEventB); - - const result = yield* writer.writeMessages([ - yield* processedMessage({ - event: REVENUE_EVENT_NAME, - eventId: sharedEventId, - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }), - yield* processedMessage({ - event: REVENUE_EVENT_NAME, - eventId: sharedEventId, - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }), - yield* processedMessage({ eventId: passEventA }), - yield* processedMessage({ eventId: passEventB }), - ]); - - // 4 messages in, but the duplicate revenue row is dropped → 3 inserted. - expect(result.messageCount).toBe(4); - expect(result.insertedRowCount).toBe(3); - - const rows = yield* findEventRows([sharedEventId]); - expect(rows.filter((row) => row.event_id === sharedEventId)).toHaveLength(1); - }), - ).pipe(Effect.provide(AnalyticsWriterService.layer), CoreAuthSession.authenticate()), - ); - - test( - "filters out events already present in ClickHouse (clickhouse_existing dedup) before insert", - withClickhouseCleanup(({ trackEvent }) => - Effect.gen(function* () { - const writer = yield* AnalyticsWriterService; - - const existingId = yield* uniqueId("existing-event"); - const freshId = yield* uniqueId("fresh-event"); - trackEvent(existingId); - trackEvent(freshId); - - // Seed `existingId` so the writer's fetchExistingEventIds sees it and - // skips it; only the fresh event should be inserted. - yield* writer.writeMessages([yield* processedMessage({ eventId: existingId })]); - - const result = yield* writer.writeMessages([ - yield* processedMessage({ eventId: existingId }), - yield* processedMessage({ eventId: freshId }), - ]); - - // 2 messages in, but `existingId` is already in ClickHouse → 1 inserted. - expect(result.messageCount).toBe(2); - expect(result.insertedRowCount).toBe(1); - - // Still exactly one row for the existing id (the duplicate was not written). - const existingRows = yield* findEventRows([existingId]); - expect(existingRows.filter((row) => row.event_id === existingId)).toHaveLength(1); - const freshRows = yield* findEventRows([freshId]); - expect(freshRows.some((row) => row.event_id === freshId)).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsWriterService.layer), CoreAuthSession.authenticate()), - ); - - test( - "deduplicates an already-present revenue event by (project_id, event_id) regardless of age (no time window)", - withClickhouseCleanup(({ trackEvent }) => - Effect.gen(function* () { - const writer = yield* AnalyticsWriterService; - - const existingRevenueId = yield* uniqueId("existing-revenue"); - const freshRevenueId = yield* uniqueId("fresh-revenue"); - trackEvent(existingRevenueId); - trackEvent(freshRevenueId); - - // Seed an existing trusted-revenue row dated WELL OUTSIDE any legacy - // 7-day safety window (90 days ago). The unbounded (project_id, event_id) - // dedup must still catch its re-dispatch — this is the at-least-once - // guarantee: a late ledger retry or duplicate provider redelivery - // collapses no matter how stale. - const nowMillis = yield* Clock.currentTimeMillis; - const ninetyDaysAgo = DateTime.toDateUtc( - DateTime.makeUnsafe(nowMillis - 90 * 24 * 60 * 60 * 1000), - ).toISOString(); - yield* writer.writeMessages([ - yield* processedMessage({ - event: REVENUE_EVENT_NAME, - eventId: existingRevenueId, - eventTimestamp: ninetyDaysAgo, - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }), - ]); - - // Second batch re-sends the (90-day-old) existing revenue id plus a fresh - // one. The already-present revenue row is not re-inserted; the fresh one is. - const result = yield* writer.writeMessages([ - yield* processedMessage({ - event: REVENUE_EVENT_NAME, - eventId: existingRevenueId, - eventTimestamp: ninetyDaysAgo, - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }), - yield* processedMessage({ - event: REVENUE_EVENT_NAME, - eventId: freshRevenueId, - eventTimestamp: (yield* DateTime.nowAsDate).toISOString(), - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }), - ]); - - expect(result.messageCount).toBe(2); - expect(result.insertedRowCount).toBe(1); - - const existingRows = yield* findEventRows([existingRevenueId]); - expect(existingRows.filter((row) => row.event_id === existingRevenueId)).toHaveLength(1); - const freshRows = yield* findEventRows([freshRevenueId]); - expect(freshRows.some((row) => row.event_id === freshRevenueId)).toBe(true); - }), - ).pipe(Effect.provide(AnalyticsWriterService.layer), CoreAuthSession.authenticate()), - ); - - test( - "returns insertedRowCount=0 and messageCount=0 for an empty batch and writes nothing", - // No cleanup wrapper: an empty batch writes no rows. - Effect.gen(function* () { - const writer = yield* AnalyticsWriterService; - const result = yield* writer.writeMessages([]); - expect(result.insertedRowCount).toBe(0); - expect(result.messageCount).toBe(0); - }).pipe(Effect.provide(AnalyticsWriterService.layer), CoreAuthSession.authenticate()), - ); - - vitestTest.todo( - "ClickhouseError branch: wraps a ClickHouse insert failure as AnalyticsWriterServiceError and writes nothing for the bad row. Deferred — there is no schema-valid `writeMessages` input that deterministically makes the live ClickHouse Cloud instance reject an `insertBatch`. The previous attempt (an out-of-Int32 `source_partition` of 9_999_999_999) was silently accepted by JSONEachRow on this instance (the insert succeeded with insertedRowCount=1), and a far-future DateTime64 is likewise tolerated here (the sibling AnalyticsJanitorService DateTime64-ceiling case returned a normal result in the same run). Because every field flows through `AnalyticsWriterMessageType` schema validation, no wrong-typed/NULL value can reach a non-Nullable column, and the only remaining way to force a ClickhouseError is a fault-injecting Clickhouse double, which the integration tier forbids. The catchTags(ClickhouseError → AnalyticsWriterServiceError) wrapping is exercised at the unit tier instead.", - ); - - vitestTest.todo( - "DatabaseError branch: wraps a MySQL failure during project→organization resolution. Deferred — `resolveOrganizationByProject` only fails if the live `project` SELECT errors (e.g. connection loss / dropped table); there is no in-process seam to inject a DatabaseError through the public `writeMessages` API without faking the Db layer, which integration tests forbid. The success-path org resolution is already covered above.", - ); -}); diff --git a/packages/core/test/services/analyticsIngest/AnalyticsWriterService.test.ts b/packages/core/test/services/analyticsIngest/AnalyticsWriterService.test.ts deleted file mode 100644 index 18d1237d9..000000000 --- a/packages/core/test/services/analyticsIngest/AnalyticsWriterService.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { constant } from "@voidhash/lib/lang"; -import { describe, expect, it } from "vite-plus/test"; - -import { REVENUE_TRUSTED_SOURCE_TOPIC } from "../../../src/domain/internalAnalytics/InternalAnalyticsEvents.ts"; -import { - dedupeRevenueRowsWithinBatch, - isRevenueAnalyticsWriterRow, -} from "../../../src/services/analyticsIngest/AnalyticsWriterService.ts"; - -/** - * A trusted revenue writer row: stamped with the revenue source topic and a - * reserved revenue event name (sourced from the domain so the test exercises - * the real {@link isReservedRevenueEventName} integration, not a string copy). - */ -const revenueRow = (overrides: Record = {}): Record => ({ - source_topic: REVENUE_TRUSTED_SOURCE_TOPIC, - event_name: "$purchase.completed", - event_id: "evt-1", - ...overrides, -}); - -/** A customer-SDK row that must always pass through untouched. */ -const nonRevenueRow = (overrides: Record = {}): Record => ({ - source_topic: "capture.v1", - event_name: "checkout_started", - event_id: "evt-non-1", - ...overrides, -}); - -describe("isRevenueAnalyticsWriterRow", () => { - it("returns true for a row with the revenue topic and a reserved revenue event name", () => { - expect(isRevenueAnalyticsWriterRow(revenueRow())).toBe(true); - }); - - it("returns true for every reserved revenue event name on the revenue topic", () => { - // Drive the real domain set so adding a new reserved event keeps coverage. - const reservedNames = constant([ - "$purchase.refunded", - "$purchase.revoked", - "$subscription.created", - "$subscription.renewed", - "$subscription.canceled", - "$subscription.transferred_in", - ]); - for (const event_name of reservedNames) { - expect(isRevenueAnalyticsWriterRow(revenueRow({ event_name }))).toBe(true); - } - }); - - it("returns false for a non-revenue source topic even with a reserved event name", () => { - expect(isRevenueAnalyticsWriterRow(revenueRow({ source_topic: "capture.v1" }))).toBe(false); - }); - - it("returns false when source_topic is missing", () => { - const { source_topic, ...rest } = revenueRow(); - void source_topic; - expect(isRevenueAnalyticsWriterRow(rest)).toBe(false); - }); - - it("returns false when event_name is not a reserved revenue event name", () => { - expect(isRevenueAnalyticsWriterRow(revenueRow({ event_name: "purchase.completed" }))).toBe( - false, - ); - }); - - it("returns false when event_name is not a string", () => { - expect(isRevenueAnalyticsWriterRow(revenueRow({ event_name: 42 }))).toBe(false); - expect(isRevenueAnalyticsWriterRow(revenueRow({ event_name: undefined }))).toBe(false); - }); -}); - -describe("dedupeRevenueRowsWithinBatch", () => { - it("removes duplicate revenue event_ids within the batch, first seen wins", () => { - const first = revenueRow({ event_id: "evt-dup", event_name: "$purchase.completed" }); - const second = revenueRow({ event_id: "evt-dup", event_name: "$purchase.refunded" }); - const result = dedupeRevenueRowsWithinBatch([first, second]); - expect(result.rows).toEqual([first]); - expect(result.rows[0]).toBe(first); - expect(result.skippedCount).toBe(1); - }); - - it("keeps non-revenue rows and revenue rows without an event_id unchanged", () => { - const noId = revenueRow({ event_id: undefined }); - const nonRev = nonRevenueRow(); - const rows = [noId, nonRev]; - const result = dedupeRevenueRowsWithinBatch(rows); - expect(result.rows).toEqual(rows); - expect(result.skippedCount).toBe(0); - }); - - it("does not dedupe non-revenue rows that share an event_id", () => { - const rows = [nonRevenueRow({ event_id: "evt-x" }), nonRevenueRow({ event_id: "evt-x" })]; - const result = dedupeRevenueRowsWithinBatch(rows); - expect(result.rows).toEqual(rows); - expect(result.skippedCount).toBe(0); - }); - - it("counts every deduplicated revenue row in skippedCount", () => { - const rows = [ - revenueRow({ event_id: "evt-a" }), - revenueRow({ event_id: "evt-a" }), - revenueRow({ event_id: "evt-a" }), - revenueRow({ event_id: "evt-b" }), - ]; - const result = dedupeRevenueRowsWithinBatch(rows); - expect(result.rows.map((r) => r.event_id)).toEqual(["evt-a", "evt-b"]); - expect(result.skippedCount).toBe(2); - }); - - it("returns the original rows array reference when skippedCount is 0 (no allocation)", () => { - const rows = [ - revenueRow({ event_id: "evt-a" }), - revenueRow({ event_id: "evt-b" }), - nonRevenueRow(), - ]; - const result = dedupeRevenueRowsWithinBatch(rows); - expect(result.skippedCount).toBe(0); - expect(result.rows).toBe(rows); - }); - - it("returns an empty result for an empty batch", () => { - const rows: Array> = []; - const result = dedupeRevenueRowsWithinBatch(rows); - expect(result.rows).toBe(rows); - expect(result.skippedCount).toBe(0); - }); - - it("treats a non-string event_id as no id (passes through, never deduped)", () => { - const rows = [revenueRow({ event_id: 7 }), revenueRow({ event_id: 7 })]; - const result = dedupeRevenueRowsWithinBatch(rows); - expect(result.rows).toEqual(rows); - expect(result.skippedCount).toBe(0); - }); -}); diff --git a/packages/core/test/services/analyticsIngest/DlqProducer.integration.test.ts b/packages/core/test/services/analyticsIngest/DlqProducer.integration.test.ts deleted file mode 100644 index 6dcdf8f4c..000000000 --- a/packages/core/test/services/analyticsIngest/DlqProducer.integration.test.ts +++ /dev/null @@ -1,365 +0,0 @@ -/** - * Integration tests for {@link DlqProducer}, run against the real backend stack - * provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB). - * - * `DlqProducer` is the thin processor-side adapter that turns the wire-stable - * {@link EventProcessorDlqV1} envelopes into `analytics_ingest_dlq` rows via - * {@link AnalyticsIngestDlqService}. The behaviour worth proving is all in the - * *persisted* row, so every test reads the written DLQ row straight back from - * MySQL rather than trusting the (void) return value: - * - `lane` → `route_class` mapping (overflow/historical pass through, anything - * else collapses to `main`), - * - the `:`-delimited `sourceOffset` suffix parsed into `source_sequence` - * (with a `0` fallback when the suffix is not a number), - * - `rawValue` JSON parsed into `payload_json` (falling back to the event - * itself when it is absent / unparseable), - * - `attempt_count` always seeded to 0, - * - fire-and-forget `forEach({ discard: true })` semantics: every event in a - * batch is written even though the producer returns void. - * - * Conventions: this service takes no `AuthSession` (the harness's default - * authenticate is a no-op here but kept for parity with the other suites), and - * the only parent row it needs — the project — is the seeded - * {@link CoreTestFixture} container, so tests just stamp unique `captureId`s - * (the table's UNIQUE key) per call and self-clean via {@link withDlqCleanup}. - * The DLQ rows carry no append-only audit entries, so cleanup deletes the rows - * alone. - */ -import { Clock, DateTime, Effect, Schema } from "effect"; -import { describe, expect, test as vitestTest } from "vitest"; - -import { DlqProducer } from "@voidhash/core/services/analyticsIngest/DlqProducer"; -import { AnalyticsIngestDlqService } from "@voidhash/core/services/analyticsIngest/AnalyticsIngestDlqService"; -import type { EventProcessorDlqV1 } from "@voidhash/core/domain/analyticsIngest/AnalyticsIngest"; -import { Db, analyticsIngestDlq, inArray } from "@voidhash/db"; - -import { CoreAuthSession } from "@testing/CoreAuthSession"; -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; -import { CoreTestFixture } from "@testing/CoreTestFixture"; - -const { test } = CoreIntegrationTestHarness.make(); - -const projectId = CoreTestFixture.projectId; - -const encodeJson = Schema.encodeSync(Schema.UnknownFromJsonString); - -/** Reads the `failureId` out of a stored `payload_json` blob, if present. */ -const failureIdOf = (payloadJson: unknown): string | undefined => { - if (typeof payloadJson !== "object" || payloadJson === null) { - return undefined; - } - if (!("failureId" in payloadJson)) { - return undefined; - } - const { failureId } = payloadJson; - if (typeof failureId !== "string") { - return undefined; - } - return failureId; -}; - -/** Monotonic counter so capture ids stay unique even within the same millisecond. */ -let seq = 0; -const uniqueCaptureId = (label: string) => - Effect.map(Clock.currentTimeMillis, (now) => `it-dlq-${label}-${now}-${seq++}`); - -/** - * Build a fully-populated {@link EventProcessorDlqV1} envelope, letting each - * test override only the fields it asserts on. Defaults are valid wire values - * so a test that doesn't care about, say, the lane still produces a writable row. - */ -const dlqEvent = ( - overrides: Partial = {}, -): Effect.Effect => - Effect.gen(function* () { - return { - captureId: yield* uniqueCaptureId("evt"), - distinctId: "distinct-1", - failedAt: (yield* DateTime.nowAsDate).toISOString(), - failureClass: "schema_rejected", - failureId: yield* uniqueCaptureId("failure"), - failureMessage: "schema validation failed", - headers: {}, - lane: "main", - projectId, - schemaVersion: 1, - sourceOffset: "topic-0:42", - sourcePartition: 0, - sourceTopic: "analytics.main.0", - ...overrides, - }; - }); - -/** Read a DLQ row straight from the database by its capture id (UNIQUE). */ -const findDlqRowByCaptureId = (captureId: string) => - Effect.gen(function* () { - const db = yield* Db; - return yield* db.query.analyticsIngestDlq.findFirst({ - where: { captureId }, - }); - }); - -/** - * Delete the DLQ rows created by a test. The table carries no append-only audit - * trail, so the rows are the only artifacts. Each delete is `ignore`d so a - * missing row never turns the finalizer into a failure. - */ -const cleanupDlqRows = (captureIds: ReadonlyArray) => - Effect.gen(function* () { - if (captureIds.length === 0) return; - const db = yield* Db; - yield* db - .delete(analyticsIngestDlq) - .where(inArray(analyticsIngestDlq.captureId, [...captureIds])) - .pipe(Effect.ignore); - }); - -/** - * Wrap a test body so every DLQ row it writes is removed afterward, regardless - * of how the test exits. Pass each written `captureId` to the `track` callback; - * cleanup reads the collected ids lazily at finalization via `Effect.ensuring`, - * so it sees every id tracked while the body ran (including on failure). - */ -const withDlqCleanup = ( - body: (track: (captureId: string) => void) => Effect.Effect, -): Effect.Effect => { - const captureIds: string[] = []; - return body((captureId) => { - captureIds.push(captureId); - }).pipe(Effect.ensuring(cleanupDlqRows(captureIds))); -}; - -describe("DlqProducer.dbLive publishBatch", () => { - test( - "maps overflow/historical lanes through and collapses everything else to main", - withDlqCleanup((track) => - Effect.gen(function* () { - const producer = yield* DlqProducer; - - const overflow = yield* dlqEvent({ captureId: yield* uniqueCaptureId("overflow"), lane: "overflow" }); - const historical = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("historical"), - lane: "historical", - }); - const main = yield* dlqEvent({ captureId: yield* uniqueCaptureId("main"), lane: "main" }); - const unknown = yield* dlqEvent({ captureId: yield* uniqueCaptureId("unknown"), lane: "unknown" }); - track(overflow.captureId!); - track(historical.captureId!); - track(main.captureId!); - track(unknown.captureId!); - - yield* producer.publishBatch([overflow, historical, main, unknown]); - - const overflowRow = yield* findDlqRowByCaptureId(overflow.captureId!); - expect(overflowRow?.routeClass).toBe("overflow"); - const historicalRow = yield* findDlqRowByCaptureId(historical.captureId!); - expect(historicalRow?.routeClass).toBe("historical"); - const mainRow = yield* findDlqRowByCaptureId(main.captureId!); - expect(mainRow?.routeClass).toBe("main"); - // "unknown" is not overflow/historical, so it must collapse to "main". - const unknownRow = yield* findDlqRowByCaptureId(unknown.captureId!); - expect(unknownRow?.routeClass).toBe("main"); - }), - ).pipe( - Effect.provide(DlqProducer.dbLive), - Effect.provide(AnalyticsIngestDlqService.layer), - CoreAuthSession.authenticate(), - ), - ); - - test( - "parses the colon-delimited sourceOffset suffix into source_sequence and falls back to 0", - withDlqCleanup((track) => - Effect.gen(function* () { - const producer = yield* DlqProducer; - - const numericSuffix = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("seq-num"), - sourceOffset: "analytics.main.0:1234", - }); - const nonNumericSuffix = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("seq-nan"), - sourceOffset: "offset-without-number", - }); - track(numericSuffix.captureId!); - track(nonNumericSuffix.captureId!); - - yield* producer.publishBatch([numericSuffix, nonNumericSuffix]); - - const numericRow = yield* findDlqRowByCaptureId(numericSuffix.captureId!); - expect(numericRow?.sourceSequence).toBe(1234); - // No parseable trailing number → fallback 0. - const fallbackRow = yield* findDlqRowByCaptureId(nonNumericSuffix.captureId!); - expect(fallbackRow?.sourceSequence).toBe(0); - }), - ).pipe( - Effect.provide(DlqProducer.dbLive), - Effect.provide(AnalyticsIngestDlqService.layer), - CoreAuthSession.authenticate(), - ), - ); - - test( - "parses rawValue JSON into payload_json and falls back to the event when absent or unparseable", - withDlqCleanup((track) => - Effect.gen(function* () { - const producer = yield* DlqProducer; - - const validJson = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("payload-json"), - rawValue: encodeJson({ marker: "decoded", nested: { ok: true } }), - }); - const unparseable = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("payload-bad"), - rawValue: "{not valid json", - }); - const { rawValue: _absentRawValue, ...absent } = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("payload-absent"), - }); - track(validJson.captureId!); - track(unparseable.captureId!); - track(absent.captureId!); - - yield* producer.publishBatch([validJson, unparseable, absent]); - - const decodedRow = yield* findDlqRowByCaptureId(validJson.captureId!); - expect(decodedRow?.payloadJson).toEqual({ marker: "decoded", nested: { ok: true } }); - - // Unparseable rawValue → the producer stores the original event object. - const fallbackRow = yield* findDlqRowByCaptureId(unparseable.captureId!); - expect(failureIdOf(fallbackRow?.payloadJson)).toBe(unparseable.failureId); - - // Missing rawValue → likewise stores the original event object. - const absentRow = yield* findDlqRowByCaptureId(absent.captureId!); - expect(failureIdOf(absentRow?.payloadJson)).toBe(absent.failureId); - }), - ).pipe( - Effect.provide(DlqProducer.dbLive), - Effect.provide(AnalyticsIngestDlqService.layer), - CoreAuthSession.authenticate(), - ), - ); - - test( - "seeds attempt_count to 0 and carries through capture/distinct/failure/shard fields", - withDlqCleanup((track) => - Effect.gen(function* () { - const producer = yield* DlqProducer; - - const event = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("fields"), - distinctId: "distinct-fields", - failureClass: "policy_rejected", - failureMessage: "rejected by policy", - sourceTopic: "analytics.main.7", - }); - track(event.captureId!); - - yield* producer.publishBatch([event]); - - const row = yield* findDlqRowByCaptureId(event.captureId!); - expect(row).toBeDefined(); - expect(row?.attemptCount).toBe(0); - expect(row?.captureId).toBe(event.captureId); - expect(row?.distinctId).toBe("distinct-fields"); - expect(row?.failureClass).toBe("policy_rejected"); - expect(row?.failureMessage).toBe("rejected by policy"); - // `sourceTopic` is recorded as the DLQ row's `source_shard`. - expect(row?.sourceShard).toBe("analytics.main.7"); - expect(row?.projectId).toBe(projectId); - }), - ).pipe( - Effect.provide(DlqProducer.dbLive), - Effect.provide(AnalyticsIngestDlqService.layer), - CoreAuthSession.authenticate(), - ), - ); - - test( - "writes every event in a batch (fire-and-forget forEach, returns void)", - withDlqCleanup((track) => - Effect.gen(function* () { - const producer = yield* DlqProducer; - - const events = yield* Effect.all( - Array.from({ length: 3 }, (_, i) => - uniqueCaptureId(`batch-${i}`).pipe( - Effect.flatMap((captureId) => dlqEvent({ captureId })), - ), - ), - ); - for (const event of events) track(event.captureId!); - - const result = yield* producer.publishBatch(events); - expect(result).toBeUndefined(); - - for (const event of events) { - const row = yield* findDlqRowByCaptureId(event.captureId!); - expect(row).toBeDefined(); - expect(row?.captureId).toBe(event.captureId); - } - }), - ).pipe( - Effect.provide(DlqProducer.dbLive), - Effect.provide(AnalyticsIngestDlqService.layer), - CoreAuthSession.authenticate(), - ), - ); - - test( - "defaults a missing projectId to 'unknown'", - withDlqCleanup((track) => - Effect.gen(function* () { - const producer = yield* DlqProducer; - - const { projectId: _eventProjectId, ...event } = yield* dlqEvent({ - captureId: yield* uniqueCaptureId("no-project"), - }); - track(event.captureId!); - - yield* producer.publishBatch([event]); - - const row = yield* findDlqRowByCaptureId(event.captureId!); - expect(row?.projectId).toBe("unknown"); - }), - ).pipe( - Effect.provide(DlqProducer.dbLive), - Effect.provide(AnalyticsIngestDlqService.layer), - CoreAuthSession.authenticate(), - ), - ); -}); - -describe("DlqProducer.noop publishBatch", () => { - test( - "returns void and writes nothing", - // No cleanup wrapper: the noop layer performs no DB writes. - Effect.gen(function* () { - const producer = yield* DlqProducer; - - const captureId = yield* uniqueCaptureId("noop"); - const event = yield* dlqEvent({ captureId }); - - const result = yield* producer.publishBatch([event]); - expect(result).toBeUndefined(); - - const row = yield* findDlqRowByCaptureId(captureId); - expect(row).toBeUndefined(); - }).pipe(Effect.provide(DlqProducer.noop), CoreAuthSession.authenticate()), - ); -}); - -// `publishBatch` wraps any `AnalyticsIngestDlqServiceError` from -// `recordFailure` in a `DlqProducerError` ("failed to record analytics ingest -// DLQ row"). Triggering that failure deterministically needs a real -// `recordFailure`/DB error, but the only failure seam is the INSERT itself: -// `routeClass`/`projectId` are always coerced to valid values and the UNIQUE -// `capture_id` collision is absorbed by `onDuplicateKeyUpdate`, so no input -// reliably makes the live PlanetScale/Vitess INSERT throw (overflow on the -// varchar columns truncates rather than errors, and is mode-dependent). -// Faking `AnalyticsIngestDlqService` to throw is disallowed (no mocks of -// services we can run). Deferred until there is an in-process fault-injection -// seam on `Db`/`AnalyticsIngestDlqService`. -vitestTest.todo( - "DlqProducer.dbLive publishBatch wraps a recordFailure failure as DlqProducerError", -); diff --git a/packages/core/test/services/analyticsIngest/EventCaptureService.integration.test.ts b/packages/core/test/services/analyticsIngest/EventCaptureService.integration.test.ts deleted file mode 100644 index 96192bc6c..000000000 --- a/packages/core/test/services/analyticsIngest/EventCaptureService.integration.test.ts +++ /dev/null @@ -1,598 +0,0 @@ -/** - * Integration tests for {@link EventCaptureService}, run against the real - * backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB). The service resolves its caller from the *capture token* in - * the `api_key` / `capture_project_policy` tables rather than from an - * {@link AuthSession}, so these tests authenticate by seeding a public api-key - * row and never call {@link CoreAuthSession.authenticate}. - * - * What is real vs. doubled: - * - `Db` is the live MySQL connection: every test seeds a `vh_pk_*` public - * api-key (and optionally a `capture_project_policy`) under the shared - * fixture project and verifies the service resolves them in one query. - * - `PolicyCounterStore` and `CaptureIngress` are *port boundaries* whose - * concrete implementations are Cloudflare-runtime backed (KV / Durable - * Objects) with no in-process seam. The port ships a sanctioned - * {@link PolicyCounterStore.noop} layer; we use it for the allow path and - * inject small typed doubles to exercise the reject / quota / publish paths. - * - The `PolicyCounterStore` port colors its effects with the - * {@link PlatformRuntime} marker, so the harness's service set is not - * sufficient on its own; we provide a minimal platform runtime stub - * (the noop / double counter implementations never read it). - * - * The service writes NOTHING to the database — it only reads the api-key/policy - * rows and hands accepted envelopes to the ingress port — so the persisted - * side-effect we verify is the *batch published to the ingress double*. The - * only DB rows that need cleanup are the api-key / policy rows the test seeds; - * {@link withCaptureCleanup} removes them on exit, success or failure. - * - * Typed failures are asserted with `Effect.flip` (project convention), narrowed - * with `instanceof` before reading their fields, and each failure path is - * paired with a state assertion (no batch published). - */ -import type { RouteClass } from "@voidhash/core/domain/analyticsIngest/AnalyticsIngest"; -import { - CaptureIngress, - CaptureIngressError, - type CaptureRequest, - EventCaptureService, - EventCaptureServiceError, - PolicyCounterStore, - type PolicyCounterStoreShape, - PolicyStoreError, - type PublishableCaptureEvent, -} from "@voidhash/core/services"; -import { - CaptureRateLimitedError, - CaptureUnauthorizedError, - type CaptureEvent, -} from "@voidhash/api-contracts/event-capture"; -import { apiKeys, captureProjectPolicies, Db, eq, inArray } from "@voidhash/db"; -import { Clock, DateTime, Effect, Layer } from "effect"; -import { describe, expect } from "vitest"; -import { PlatformRuntime } from "@voidhash/platform/PlatformRuntime"; - -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; -import { CoreTestFixture } from "@testing/CoreTestFixture"; - -const { test } = CoreIntegrationTestHarness.make(); - -const projectId = CoreTestFixture.projectId; - -/** Monotonic counter so tokens/ids stay unique even within the same millisecond. */ -let seq = 0; -const uniqueToken = (label: string) => - Effect.map(Clock.currentTimeMillis, (now) => `vh_pk_it${label}${now}${seq++}`); -const uniqueId = (label: string) => - Effect.map(Clock.currentTimeMillis, (now) => `it_capt_${label}_${now}_${seq++}`); - -/** - * Minimal {@link PlatformRuntime} stub. The noop / double {@link PolicyCounterStore} - * implementations never read it; it exists only to discharge the runtime-phase - * marker the port colors its effects with. - */ -const PlatformRuntimeStub = Layer.succeed(PlatformRuntime, PlatformRuntime.of({})); - -/** - * A {@link CaptureIngress} double that records every batch handed to it, so a - * test can assert which envelopes were published (and on which route). Returns - * the layer plus the live `batches` array it appends to. - */ -const makeIngressSpy = () => { - const batches: Array> = []; - const layer = Layer.succeed(CaptureIngress, { - enqueueBatch: (events) => - Effect.sync(() => { - batches.push(events); - }), - }); - return { batches, layer }; -}; - -/** Flatten the recorded batches into a single list of published events. */ -const publishedEvents = ( - batches: ReadonlyArray>, -): ReadonlyArray => batches.flat(); - -/** Build a `PolicyCounterStore` layer from a partial shape, defaulting to allow-all. */ -const policyStoreLayer = (overrides: Partial = {}) => - Layer.succeed(PolicyCounterStore, { - checkRequestLimit: () => Effect.succeed({ allowed: true }), - checkEventQuota: () => Effect.succeed(true), - ...overrides, - }); - -/** - * Pipeable that wires the service-under-test for a test. The - * {@link EventCaptureService} layer is built with the per-test ingress + - * policy-store port doubles supplied (its build-time requirements), and the - * {@link PlatformRuntime} stub is layered in for the runtime-phase marker the - * policy port colors its `captureEvents` effect with. `Db` (and the other - * harness services) is left to the harness. - */ -const provideService = - (ports: { - readonly ingress: Layer.Layer; - readonly policyStore?: Layer.Layer; - }) => - (effect: Effect.Effect) => - effect.pipe( - Effect.provide( - EventCaptureService.layer.pipe( - Layer.provide(Layer.mergeAll(ports.ingress, ports.policyStore ?? policyStoreLayer())), - ), - ), - Effect.provide(PlatformRuntimeStub), - ); - -/** A fresh capture event with sensible defaults; override fields per test. */ -const captureEvent = ( - overrides: Partial = {}, -): Effect.Effect => - Effect.gen(function* () { - return { - uuid: yield* uniqueId("evt"), - event: "page_view", - context: {}, - properties: {}, - distinct_id: "user-1", - ...overrides, - }; - }); - -/** A fresh capture request wrapping the given events. */ -const captureRequest = ( - token: string, - events: ReadonlyArray, -): Effect.Effect => - Effect.gen(function* () { - const now = yield* DateTime.nowAsDate; - return { - request: { - headers: { "user-agent": "integration-test" }, - receivedAt: now, - requestId: yield* uniqueId("req"), - sentAt: now, - token, - }, - events, - }; - }); - -/** Insert a public api-key row (the capture token) under the fixture project. */ -const insertPublicApiKey = (token: string) => - Effect.gen(function* () { - const db = yield* Db; - const id = yield* uniqueId("key"); - yield* db.insert(apiKeys).values({ - end: token.slice(-8), - id, - isPublic: true, - key: token, - name: "Integration capture key", - prefix: "api_pk", - projectId, - }); - return id; - }); - -interface CapturePolicyInput { - readonly ingestEnabled?: boolean; - readonly requestsPerMinute?: number; - readonly eventsPerDay?: number; - readonly forceRoute?: string; - readonly skipEnrichment?: boolean; -} - -/** - * Only the columns the caller actually set — an undefined key must not - * overwrite an existing row's value on conflict. - */ -const definedPolicyColumns = (policy: CapturePolicyInput): CapturePolicyInput => { - const set: { - ingestEnabled?: boolean; - requestsPerMinute?: number; - eventsPerDay?: number; - forceRoute?: string; - skipEnrichment?: boolean; - } = {}; - if (policy.ingestEnabled !== undefined) set.ingestEnabled = policy.ingestEnabled; - if (policy.requestsPerMinute !== undefined) set.requestsPerMinute = policy.requestsPerMinute; - if (policy.eventsPerDay !== undefined) set.eventsPerDay = policy.eventsPerDay; - if (policy.forceRoute !== undefined) set.forceRoute = policy.forceRoute; - if (policy.skipEnrichment !== undefined) set.skipEnrichment = policy.skipEnrichment; - return set; -}; - -/** Insert (or upsert) the capture-project policy row for the fixture project. */ -const upsertCapturePolicy = (policy: CapturePolicyInput) => - Effect.gen(function* () { - const db = yield* Db; - yield* db - .insert(captureProjectPolicies) - .values({ projectId, ...policy }) - .onConflictDoUpdate({ - target: captureProjectPolicies.projectId, - set: definedPolicyColumns(policy), - }); - }); - -/** - * Delete the api-key rows and the capture-policy row the test seeded. The - * policy row is keyed by `projectId`, so it is cleared by project id; each - * delete is `ignore`d so a missing row never turns the finalizer into a - * failure. - */ -const cleanup = (apiKeyIds: ReadonlyArray, seededPolicy: boolean) => - Effect.gen(function* () { - const db = yield* Db; - if (apiKeyIds.length > 0) { - yield* db - .delete(apiKeys) - .where(inArray(apiKeys.id, [...apiKeyIds])) - .pipe(Effect.ignore); - } - if (seededPolicy) { - yield* db - .delete(captureProjectPolicies) - .where(eq(captureProjectPolicies.projectId, projectId)) - .pipe(Effect.ignore); - } - }); - -/** - * Wrap a test body so every api-key (and the policy row) it seeds is removed - * afterward, regardless of how the test exits. `trackKey` collects api-key ids; - * `markPolicy` flags that the project policy row was written. Cleanup reads the - * collected ids lazily at finalization via `Effect.ensuring`. - */ -const withCaptureCleanup = ( - body: (trackKey: (id: string) => void, markPolicy: () => void) => Effect.Effect, -): Effect.Effect => { - const apiKeyIds: string[] = []; - let seededPolicy = false; - return body( - (id) => { - apiKeyIds.push(id); - }, - () => { - seededPolicy = true; - }, - ).pipe(Effect.ensuring(Effect.suspend(() => cleanup(apiKeyIds, seededPolicy)))); -}; - -describe("EventCaptureService.captureEvents", () => { - test( - "resolves a public token, publishes envelopes on the main route, and returns accepted counts", - (() => { - const ingress = makeIngressSpy(); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("ok"); - trackKey(yield* insertPublicApiKey(token)); - - const eventA = yield* captureEvent({ event: "signup", distinct_id: "user-a" }); - const eventB = yield* captureEvent({ event: "page_view", distinct_id: "user-b" }); - - const request = yield* captureRequest(token, [eventA, eventB]); - const result = yield* service.captureEvents(request); - - expect(result.accepted).toBe(2); - expect(result.rejected).toBe(0); - - const events = publishedEvents(ingress.batches); - expect(events.length).toBe(2); - // No policy row → default policy → "main" route for within-quota events. - expect(events.every((entry) => entry.routeClass === "main")).toBe(true); - expect(events.some((entry) => entry.envelope.event === "signup")).toBe(true); - expect(events.some((entry) => entry.envelope.event === "page_view")).toBe(true); - // Envelopes carry the resolved project + the trimmed token. - expect(events.every((entry) => entry.envelope.projectId === projectId)).toBe(true); - expect(events.every((entry) => entry.envelope.token === token)).toBe(true); - }), - ).pipe(provideService({ ingress: ingress.layer })); - })(), - ); - - test( - "trims whitespace around an otherwise-valid token before resolving it", - (() => { - const ingress = makeIngressSpy(); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("trim"); - trackKey(yield* insertPublicApiKey(token)); - - // Surrounding whitespace must be stripped so the DB lookup hits. - const result = yield* service.captureEvents( - yield* captureRequest(` ${token}\n`, [yield* captureEvent()]), - ); - - expect(result.accepted).toBe(1); - expect(publishedEvents(ingress.batches).length).toBe(1); - }), - ).pipe(provideService({ ingress: ingress.layer })); - })(), - ); - - test( - "fails with CaptureUnauthorizedError for a malformed token and publishes nothing", - (() => { - const ingress = makeIngressSpy(); - // No cleanup wrapper: a malformed token never reaches an insert. - return Effect.gen(function* () { - const service = yield* EventCaptureService; - - const error = yield* Effect.flip( - service.captureEvents( - yield* captureRequest("not-a-valid-token", [yield* captureEvent()]), - ), - ); - expect(error).toBeInstanceOf(CaptureUnauthorizedError); - if (error instanceof CaptureUnauthorizedError) { - expect(error.code).toBe("unauthorized"); - } - - expect(ingress.batches.length).toBe(0); - }).pipe(provideService({ ingress: ingress.layer })); - })(), - ); - - test( - "fails with CaptureUnauthorizedError for a well-formed but unknown token", - (() => { - const ingress = makeIngressSpy(); - return Effect.gen(function* () { - const service = yield* EventCaptureService; - - // Correct `vh_pk_*` shape, but no matching api-key row exists. - const error = yield* Effect.flip( - service.captureEvents( - yield* captureRequest(yield* uniqueToken("missing"), [yield* captureEvent()]), - ), - ); - expect(error).toBeInstanceOf(CaptureUnauthorizedError); - - expect(ingress.batches.length).toBe(0); - }).pipe(provideService({ ingress: ingress.layer })); - })(), - ); - - test( - "rejects the request with CaptureRateLimitedError when the project's policy disables ingest", - (() => { - const ingress = makeIngressSpy(); - return withCaptureCleanup((trackKey, markPolicy) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("disabled"); - trackKey(yield* insertPublicApiKey(token)); - yield* upsertCapturePolicy({ ingestEnabled: false }); - markPolicy(); - - const error = yield* Effect.flip( - service.captureEvents(yield* captureRequest(token, [yield* captureEvent()])), - ); - expect(error).toBeInstanceOf(CaptureRateLimitedError); - if (error instanceof CaptureRateLimitedError) { - expect(error.code).toBe("rate_limited"); - } - - expect(ingress.batches.length).toBe(0); - }), - ).pipe(provideService({ ingress: ingress.layer })); - })(), - ); - - test( - "rejects with CaptureRateLimitedError carrying retry_after_ms when the request limit is exceeded", - (() => { - const ingress = makeIngressSpy(); - const policyStore = policyStoreLayer({ - checkRequestLimit: () => Effect.succeed({ allowed: false, retryAfterMs: 4_200 }), - }); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("ratelimited"); - trackKey(yield* insertPublicApiKey(token)); - - const error = yield* Effect.flip( - service.captureEvents(yield* captureRequest(token, [yield* captureEvent()])), - ); - expect(error).toBeInstanceOf(CaptureRateLimitedError); - if (error instanceof CaptureRateLimitedError) { - expect(error.retry_after_ms).toBe(4_200); - } - - expect(ingress.batches.length).toBe(0); - }), - ).pipe(provideService({ ingress: ingress.layer, policyStore })); - })(), - ); - - test( - "rejects reserved revenue event names from public-key capture but accepts the rest", - (() => { - const ingress = makeIngressSpy(); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("reserved"); - trackKey(yield* insertPublicApiKey(token)); - - const reserved = yield* captureEvent({ event: "$purchase.completed" }); - const allowed = yield* captureEvent({ event: "checkout_started" }); - - const request = yield* captureRequest(token, [reserved, allowed]); - const result = yield* service.captureEvents(request); - - expect(result.rejected).toBe(1); - expect(result.accepted).toBe(1); - - const events = publishedEvents(ingress.batches); - expect(events.length).toBe(1); - expect(events[0]?.envelope.event).toBe("checkout_started"); - // The reserved event must never reach the ingress. - expect(events.some((entry) => entry.envelope.event === "$purchase.completed")).toBe( - false, - ); - }), - ).pipe(provideService({ ingress: ingress.layer })); - })(), - ); - - test( - "routes over-quota events to the overflow lane while staying accepted", - (() => { - const ingress = makeIngressSpy(); - const policyStore = policyStoreLayer({ checkEventQuota: () => Effect.succeed(false) }); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("overflow"); - trackKey(yield* insertPublicApiKey(token)); - - // Quota check denies → selectRoute falls back to the overflow lane. - const request = yield* captureRequest(token, [yield* captureEvent()]); - const result = yield* service.captureEvents(request); - - expect(result.accepted).toBe(1); - expect(result.rejected).toBe(0); - - const events = publishedEvents(ingress.batches); - expect(events.length).toBe(1); - expect(events[0]?.routeClass).toBe("overflow"); - }), - ).pipe(provideService({ ingress: ingress.layer, policyStore })); - })(), - ); - - test( - "honors a forced route from the project policy", - (() => { - const ingress = makeIngressSpy(); - return withCaptureCleanup((trackKey, markPolicy) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("historical"); - trackKey(yield* insertPublicApiKey(token)); - yield* upsertCapturePolicy({ forceRoute: "historical" }); - markPolicy(); - - const request = yield* captureRequest(token, [yield* captureEvent()]); - const result = yield* service.captureEvents(request); - - expect(result.accepted).toBe(1); - const events = publishedEvents(ingress.batches); - expect(events.length).toBe(1); - expect(events[0]?.routeClass).toBe("historical"); - // Forced historical route stamps the envelope as historical. - expect(events[0]?.envelope.routing.isHistorical).toBe(true); - }), - ).pipe(provideService({ ingress: ingress.layer })); - })(), - ); - - test( - "counts a per-event quota failure as rejected and continues processing the batch", - (() => { - const ingress = makeIngressSpy(); - // The first per-event quota check fails (PolicyStoreError surfaced inside - // the per-event `Effect.result`); the second succeeds. The failing event is - // counted rejected, the loop keeps going. - let calls = 0; - const policyStore = policyStoreLayer({ - checkEventQuota: () => { - calls += 1; - if (calls === 1) { - return Effect.fail(new PolicyStoreError({ message: "per-event quota check failed" })); - } - return Effect.succeed(true); - }, - }); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("partialfail"); - trackKey(yield* insertPublicApiKey(token)); - - const result = yield* service.captureEvents( - yield* captureRequest(token, [yield* captureEvent(), yield* captureEvent()]), - ); - - expect(result.rejected).toBe(1); - expect(result.accepted).toBe(1); - expect(publishedEvents(ingress.batches).length).toBe(1); - }), - ).pipe(provideService({ ingress: ingress.layer, policyStore })); - })(), - ); - - test( - "wraps a CaptureIngressError from the publish step as EventCaptureServiceError", - (() => { - const failingIngress = Layer.succeed(CaptureIngress, { - enqueueBatch: () => - Effect.fail(new CaptureIngressError({ message: "ingress unavailable" })), - }); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("ingresserr"); - trackKey(yield* insertPublicApiKey(token)); - - const error = yield* Effect.flip( - service.captureEvents(yield* captureRequest(token, [yield* captureEvent()])), - ); - // CaptureIngressError is wrapped at the public boundary, surfacing the - // adapter's message on EventCaptureServiceError. - expect(error).toBeInstanceOf(EventCaptureServiceError); - if (error instanceof EventCaptureServiceError) { - expect(error.message).toBe("ingress unavailable"); - } - }), - ).pipe(provideService({ ingress: failingIngress })); - })(), - ); - - test( - "wraps a PolicyStoreError from the request-limit check as EventCaptureServiceError", - (() => { - const ingress = makeIngressSpy(); - const policyStore = policyStoreLayer({ - checkRequestLimit: () => - Effect.fail(new PolicyStoreError({ message: "counter store down" })), - }); - return withCaptureCleanup((trackKey) => - Effect.gen(function* () { - const service = yield* EventCaptureService; - - const token = yield* uniqueToken("policyerr"); - trackKey(yield* insertPublicApiKey(token)); - - const error = yield* Effect.flip( - service.captureEvents(yield* captureRequest(token, [yield* captureEvent()])), - ); - expect(error).toBeInstanceOf(EventCaptureServiceError); - if (error instanceof EventCaptureServiceError) { - expect(error.message).toBe("counter store down"); - } - - expect(ingress.batches.length).toBe(0); - }), - ).pipe(provideService({ ingress: ingress.layer, policyStore })); - })(), - ); -}); diff --git a/packages/core/test/services/analyticsIngest/EventCaptureService.test.ts b/packages/core/test/services/analyticsIngest/EventCaptureService.test.ts deleted file mode 100644 index 254cd22bd..000000000 --- a/packages/core/test/services/analyticsIngest/EventCaptureService.test.ts +++ /dev/null @@ -1,377 +0,0 @@ -import { ANONYMOUS_USER_ID_PREFIX } from "@voidhash/lib"; -import { - CaptureRateLimitedError, - CaptureUnauthorizedError, - type CaptureEvent, -} from "@voidhash/api-contracts/event-capture"; -import { DateTime, Effect } from "effect"; - -import { describe, expect, it } from "../../../src/testing/effect-vitest.ts"; -import type { - CaptureProjectPolicy, - RouteClass, - RouteDecision, -} from "../../../src/domain/analyticsIngest/AnalyticsIngest.ts"; -import { - CAPTURE_TOPIC_DLQ, - CAPTURE_TOPIC_HISTORICAL, - CAPTURE_TOPIC_MAIN, - CAPTURE_TOPIC_OVERFLOW, - makeEnvelope, - resolveEventTimestamp, - selectRoute, - tokenSuffix, - validateCaptureToken, -} from "../../../src/services/analyticsIngest/EventCaptureService.ts"; - -// ============================================================================= -// Fixtures (fresh per test via const-returning builders) -// ============================================================================= - -const instant = (iso: string): Date => DateTime.toDateUtc(DateTime.makeUnsafe(iso)); - -const policy = (overrides: Partial = {}): CaptureProjectPolicy => ({ - ingestEnabled: true, - projectId: "proj_123", - skipEnrichment: false, - ...overrides, -}); - -const route = (overrides: Partial = {}): RouteDecision => ({ - isHistorical: false, - routeClass: "main", - skipEnrichment: false, - targetTopic: CAPTURE_TOPIC_MAIN, - ...overrides, -}); - -const captureEvent = ( - overrides: Partial = {}, -): typeof CaptureEvent.Type => ({ - uuid: "evt_uuid_1", - event: "page_view", - context: { library: "web" }, - properties: { plan: "pro" }, - distinct_id: "user_42", - ...overrides, -}); - -const envelopeArgs = ( - overrides: { - event?: typeof CaptureEvent.Type; - route?: RouteDecision; - request?: { - readonly clientIp?: string; - readonly headers: Readonly>; - readonly path?: string; - readonly requestId: string; - }; - receivedAt?: Date; - sentAt?: Date; - token?: string; - organizationId?: string; - projectId?: string; - } = {}, -) => ({ - event: overrides.event ?? captureEvent(), - organizationId: overrides.organizationId ?? "org_1", - projectId: overrides.projectId ?? "proj_123", - receivedAt: overrides.receivedAt ?? instant("2026-01-01T00:00:05.000Z"), - request: overrides.request ?? { - headers: { "user-agent": "test-agent" }, - requestId: "req_1", - }, - route: overrides.route ?? route(), - sentAt: overrides.sentAt ?? instant("2026-01-01T00:00:02.000Z"), - token: overrides.token ?? "vh_pk_abcd1234", -}); - -// ============================================================================= -// validateCaptureToken (returns an Effect; run with Effect.runPromise / flip) -// ============================================================================= - -describe("validateCaptureToken", () => { - it.effect("accepts a token matching the vh_pk_* format and returns it", () => - Effect.gen(function* () { - const result = yield* validateCaptureToken("vh_pk_abcd1234"); - expect(result).toBe("vh_pk_abcd1234"); - }), - ); - - it.effect("trims surrounding whitespace before validating and returns the trimmed token", () => - Effect.gen(function* () { - const result = yield* validateCaptureToken(" vh_pk_abcd1234\n"); - expect(result).toBe("vh_pk_abcd1234"); - }), - ); - - it.effect("rejects an empty token with CaptureUnauthorizedError 'missing token'", () => - Effect.gen(function* () { - // `Effect.flip` moves the typed error into the success channel so we can - // assert on its concrete type without poking into the Cause. - const error = yield* Effect.flip(validateCaptureToken("")); - expect(error).toBeInstanceOf(CaptureUnauthorizedError); - expect(error.error).toBe("missing token"); - expect(error.code).toBe("unauthorized"); - }), - ); - - it.effect("treats a whitespace-only token as missing", () => - Effect.gen(function* () { - const error = yield* Effect.flip(validateCaptureToken(" ")); - expect(error).toBeInstanceOf(CaptureUnauthorizedError); - expect(error.error).toBe("missing token"); - }), - ); - - it.each([ - ["pk_abcd1234", "wrong prefix"], - ["vh_pk_", "empty body"], - ["vh_pk_abc-123", "non-word char"], - ["VH_PK_ABCD", "uppercase prefix"], - ])("rejects malformed token %j (%s) with 'invalid token format'", (token) => - Effect.runPromise( - Effect.gen(function* () { - const error = yield* Effect.flip(validateCaptureToken(token)); - expect(error).toBeInstanceOf(CaptureUnauthorizedError); - expect(error.error).toBe("invalid token format"); - expect(error.code).toBe("unauthorized"); - }), - ), - ); -}); - -// ============================================================================= -// tokenSuffix -// ============================================================================= - -describe("tokenSuffix", () => { - it("returns the last 4 characters of a token", () => { - expect(tokenSuffix("vh_pk_abcd1234")).toBe("1234"); - }); - - it("returns the whole string when shorter than 4 characters", () => { - expect(tokenSuffix("ab")).toBe("ab"); - }); -}); - -// ============================================================================= -// resolveEventTimestamp (pure) -// ============================================================================= - -describe("resolveEventTimestamp", () => { - const receivedAt = instant("2026-01-01T00:00:05.000Z"); - const sentAt = instant("2026-01-01T00:00:02.000Z"); - const timestamp = instant("2026-01-01T00:00:01.000Z"); - - it("returns the explicit timestamp when provided (highest priority)", () => { - expect(resolveEventTimestamp({ receivedAt, sentAt, timestamp })).toBe(timestamp); - }); - - it("falls back to sentAt when timestamp is missing", () => { - expect(resolveEventTimestamp({ receivedAt, sentAt })).toBe(sentAt); - }); - - it("falls back to receivedAt when both timestamp and sentAt are missing", () => { - expect(resolveEventTimestamp({ receivedAt })).toBe(receivedAt); - }); -}); - -// ============================================================================= -// selectRoute (returns an Effect) -// ============================================================================= - -describe("selectRoute", () => { - it.effect("routes to 'main'/CAPTURE_TOPIC_MAIN when not over quota and no forceRoute", () => - Effect.gen(function* () { - const decision = yield* selectRoute({ overQuota: false, policy: policy() }); - expect(decision).toStrictEqual({ - isHistorical: false, - routeClass: "main", - skipEnrichment: false, - targetTopic: CAPTURE_TOPIC_MAIN, - }); - }), - ); - - it.effect("routes to 'overflow'/CAPTURE_TOPIC_OVERFLOW when over quota", () => - Effect.gen(function* () { - const decision = yield* selectRoute({ overQuota: true, policy: policy() }); - expect(decision.routeClass).toBe("overflow"); - expect(decision.targetTopic).toBe(CAPTURE_TOPIC_OVERFLOW); - expect(decision.isHistorical).toBe(false); - }), - ); - - it.each<[Exclude, string, boolean]>([ - ["main", CAPTURE_TOPIC_MAIN, false], - ["overflow", CAPTURE_TOPIC_OVERFLOW, false], - ["historical", CAPTURE_TOPIC_HISTORICAL, true], - ["dlq", CAPTURE_TOPIC_DLQ, false], - ])( - "respects forceRoute=%s (topic %s, isHistorical=%s) even when over quota", - (forceRoute, expectedTopic, expectedHistorical) => - Effect.runPromise( - Effect.gen(function* () { - // overQuota is intentionally true to prove forceRoute wins over the quota fallback. - const decision = yield* selectRoute({ overQuota: true, policy: policy({ forceRoute }) }); - expect(decision.routeClass).toBe(forceRoute); - expect(decision.targetTopic).toBe(expectedTopic); - expect(decision.isHistorical).toBe(expectedHistorical); - }), - ), - ); - - it.effect("rejects forceRoute='custom' with CaptureRateLimitedError", () => - Effect.gen(function* () { - const error = yield* Effect.flip( - selectRoute({ overQuota: false, policy: policy({ forceRoute: "custom" }) }), - ); - expect(error).toBeInstanceOf(CaptureRateLimitedError); - expect(error.code).toBe("rate_limited"); - expect(error.error).toBe("custom routes are not supported in this deployment"); - }), - ); - - it.effect("carries skipEnrichment through from the policy into the RouteDecision", () => - Effect.gen(function* () { - const decision = yield* selectRoute({ - overQuota: false, - policy: policy({ skipEnrichment: true }), - }); - expect(decision.skipEnrichment).toBe(true); - }), - ); - - it.effect("sets isHistorical=true only for the historical route", () => - Effect.gen(function* () { - const main = yield* selectRoute({ overQuota: false, policy: policy() }); - const historical = yield* selectRoute({ - overQuota: false, - policy: policy({ forceRoute: "historical" }), - }); - expect(main.isHistorical).toBe(false); - expect(historical.isHistorical).toBe(true); - }), - ); -}); - -// ============================================================================= -// makeEnvelope (pure) -// ============================================================================= - -describe("makeEnvelope", () => { - it("builds a CapturedEventV1 with schema version, captureId, routing, ids and timestamps", () => { - const envelope = makeEnvelope(envelopeArgs()); - expect(envelope.schemaVersion).toBe(1); - expect(typeof envelope.captureId).toBe("string"); - expect(envelope.captureId.length).toBeGreaterThan(0); - expect(envelope.organizationId).toBe("org_1"); - expect(envelope.projectId).toBe("proj_123"); - expect(envelope.token).toBe("vh_pk_abcd1234"); - expect(envelope.event).toBe("page_view"); - expect(envelope.distinctId).toBe("user_42"); - expect(envelope.context).toStrictEqual({ library: "web" }); - expect(envelope.routing).toStrictEqual(route()); - // sentAt (2s) is preferred over receivedAt (5s) since no event.timestamp. - expect(envelope.eventTimestamp).toBe("2026-01-01T00:00:02.000Z"); - expect(envelope.receivedAt).toBe("2026-01-01T00:00:05.000Z"); - expect(envelope.sentAt).toBe("2026-01-01T00:00:02.000Z"); - expect(envelope.request.requestId).toBe("req_1"); - expect(envelope.request.userAgent).toBe("test-agent"); - }); - - it("uses resolveEventTimestamp: explicit event.timestamp wins over sentAt/receivedAt", () => { - const envelope = makeEnvelope( - envelopeArgs({ - event: captureEvent({ timestamp: instant("2025-12-31T23:59:00.000Z") }), - }), - ); - expect(envelope.eventTimestamp).toBe("2025-12-31T23:59:00.000Z"); - }); - - it("includes clientEventId when the event uuid is provided", () => { - const envelope = makeEnvelope(envelopeArgs({ event: captureEvent({ uuid: "evt_uuid_99" }) })); - expect(envelope.clientEventId).toBe("evt_uuid_99"); - }); - - it("includes sessionId when session_id is provided", () => { - const envelope = makeEnvelope(envelopeArgs({ event: captureEvent({ session_id: "sess_7" }) })); - expect(envelope.sessionId).toBe("sess_7"); - }); - - it("omits sessionId when session_id is absent", () => { - const envelope = makeEnvelope(envelopeArgs()); - expect(envelope.sessionId).toBeUndefined(); - }); - - it("adds $ip to canonical properties when clientIp is provided", () => { - const envelope = makeEnvelope( - envelopeArgs({ - request: { headers: { "user-agent": "ua" }, requestId: "req_2", clientIp: "1.2.3.4" }, - }), - ); - expect(envelope.properties.$ip).toBe("1.2.3.4"); - expect(envelope.request.clientIp).toBe("1.2.3.4"); - }); - - it("omits $ip when clientIp is absent", () => { - const envelope = makeEnvelope(envelopeArgs()); - expect(envelope.properties.$ip).toBeUndefined(); - expect(envelope.request.clientIp).toBeUndefined(); - }); - - it("sets $process_person_profile=true for a non-anonymous distinct id", () => { - const envelope = makeEnvelope( - envelopeArgs({ event: captureEvent({ distinct_id: "real_user" }) }), - ); - expect(envelope.properties.$process_person_profile).toBe(true); - }); - - it("sets $process_person_profile=false for an anonymous distinct id", () => { - const envelope = makeEnvelope( - envelopeArgs({ - event: captureEvent({ distinct_id: `${ANONYMOUS_USER_ID_PREFIX}abc` }), - }), - ); - expect(envelope.properties.$process_person_profile).toBe(false); - }); - - it("honors a client-supplied $process_person_profile=true for an anonymous distinct id", () => { - // An explicit `setPersonAttributes` `$set` stamps this so anonymous users - // still get a person; the envelope must not override it back to false. - const envelope = makeEnvelope( - envelopeArgs({ - event: captureEvent({ - distinct_id: `${ANONYMOUS_USER_ID_PREFIX}abc`, - properties: { $set: { age: 25 }, $process_person_profile: true }, - }), - }), - ); - expect(envelope.properties.$process_person_profile).toBe(true); - }); - - it("honors a client-supplied $process_person_profile=false for a non-anonymous distinct id", () => { - const envelope = makeEnvelope( - envelopeArgs({ - event: captureEvent({ - distinct_id: "real_user", - properties: { $process_person_profile: false }, - }), - }), - ); - expect(envelope.properties.$process_person_profile).toBe(false); - }); - - it("falls back to the distinct-id default when $process_person_profile is not a boolean", () => { - const envelope = makeEnvelope( - envelopeArgs({ - event: captureEvent({ - distinct_id: `${ANONYMOUS_USER_ID_PREFIX}abc`, - properties: { $process_person_profile: "yes" }, - }), - }), - ); - expect(envelope.properties.$process_person_profile).toBe(false); - }); -}); diff --git a/packages/core/test/services/analyticsIngest/EventProcessorService.integration.test.ts b/packages/core/test/services/analyticsIngest/EventProcessorService.integration.test.ts deleted file mode 100644 index eaeabb7ba..000000000 --- a/packages/core/test/services/analyticsIngest/EventProcessorService.integration.test.ts +++ /dev/null @@ -1,790 +0,0 @@ -/** - * Integration tests for {@link EventProcessorService}, run against the real - * backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB + ClickHouse + WorkOS; only the project schema cache is an - * in-memory stub). - * - * `processRecordToOutputs` is a single-record orchestration: it resolves the - * project + processor policy by token (real MySQL join over `api_key` → - * `project` → `capture_project_policy`), validates policy/lane/schema rules, - * runs identity resolution directly (order-agnostic; serialized at the row level - * by the identity transaction's locks), and - * returns the three downstream output streams. Each test drives the service - * end-to-end and verifies the *persisted* side effect rather than just the - * return value: - * - DLQ rejection paths write an `analytics_ingest_dlq` row with the right - * `failure_class` / `project_id` (verified by reading the row back) and - * return empty outputs. - * - the happy path writes the `person` + `person_identity` rows that identity - * resolution creates, and the returned `ProcessorOutputs` carry the matching - * processed / person / identity wire events. - * - * Conventions: - * - The fixture seeds only user/org/member/project. This service needs a - * *public* API key under the fixture project (the capture token) plus, for - * some cases, a `capture_project_policy` row; both are created per test with - * unique tokens and cleaned up via {@link withCleanup}. - * - Every record uses a unique `captureId` / `distinctId` so the unique index - * on `analytics_ingest_dlq.capture_id` never collides with a leftover row - * and assertions stay membership-based (by capture id / distinct id), never - * exact counts. - * - {@link withCleanup} deletes everything a test created (DLQ rows, persons, - * identity rows, policy rows, api keys, migration jobs) on exit, success or - * failure. The global teardown sweep is only a backstop. - * - The collaborators the service layer needs but the harness does not provide - * (`DlqProducer`, `PersonIdentityService`, and their - * transitive ports) are wired here from their real layers, so the only - * requirement reaching the harness is `Db` (plus the harness's own set). - * - Typed failures are asserted with `Effect.flip` (project convention), - * narrowing the swapped error with `instanceof` before reading its fields. - */ -import { Clock, Effect, Layer, Schema } from "effect"; -import { describe, expect } from "vitest"; - -import type { - CapturedEventV1Type, - CapturedTransportRecord, -} from "@voidhash/core/domain/analyticsIngest/AnalyticsIngest"; -import { EventProcessorService } from "@voidhash/core/services/analyticsIngest/EventProcessorService"; -import { DlqProducer } from "@voidhash/core/services/analyticsIngest/DlqProducer"; -import { AnalyticsIngestDlqService } from "@voidhash/core/services/analyticsIngest/AnalyticsIngestDlqService"; -import { CaptureIngress } from "@voidhash/core/services/analyticsIngest/CaptureIngress"; -import { PersonIdentityService } from "@voidhash/core/services/personIdentity/PersonIdentityService"; -import { IdentityProjectionPublisher } from "@voidhash/core/services/personIdentity/IdentityProjectionPublisher"; -import { REVENUE_TRUSTED_SOURCE_TOPIC } from "@voidhash/core/domain/internalAnalytics/InternalAnalyticsEvents"; -import { - Db, - analyticsIngestDlq, - and, - apiKeys, - captureProjectPolicies, - eq, - inArray, - personIdentities, - persons, - personPersonlessIdentities, -} from "@voidhash/db"; - -import { CoreAuthSession } from "@testing/CoreAuthSession"; -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; -import { CoreTestFixture } from "@testing/CoreTestFixture"; - -const { test } = CoreIntegrationTestHarness.make(); - -const projectId = CoreTestFixture.projectId; -const organizationId = CoreTestFixture.organizationId; - -/** - * Compose the service-under-test layer together with the collaborators the - * harness does not provide. After this layer, the only outstanding requirement - * is `Db` (plus the harness's own services), satisfying `R extends - * HarnessServices`. The DLQ + identity collaborators are the real layers so the - * DB side effects they produce can be verified; external publication is wired - * to its no-op variant. - */ -const TestLayer = EventProcessorService.layer.pipe( - Layer.provide(DlqProducer.dbLive), - Layer.provide(AnalyticsIngestDlqService.layer), - Layer.provide(CaptureIngress.noop), - Layer.provide(PersonIdentityService.layer), - Layer.provide(IdentityProjectionPublisher.noop), -); - -/** Monotonic counter so ids stay unique even within the same millisecond. */ -let seq = 0; -const unique = (label: string) => - Effect.map(Clock.currentTimeMillis, (now) => `it-ep-${label}-${now}-${seq++}`); - -const encodeJson = Schema.encodeSync(Schema.UnknownFromJsonString); - -/** The route class each lane maps onto. */ -const routeClassOf = ( - lane: "main" | "overflow" | "historical", -): CapturedEventV1Type["routing"]["routeClass"] => { - if (lane === "historical") { - return "historical"; - } - if (lane === "overflow") { - return "overflow"; - } - return "main"; -}; - -const NO_PERSON_ROWS: ReadonlyArray<{ personId: string }> = []; - -interface RecordOverrides { - readonly token: string; - readonly distinctId?: string; - readonly captureId?: string; - readonly event?: string; - readonly lane?: "main" | "overflow" | "historical"; - readonly sourceTopic?: string; - readonly properties?: CapturedEventV1Type["properties"]; -} - -/** - * Build a `main`-lane {@link CapturedTransportRecord} whose routing satisfies - * the built-in processor validation rules (target topic matches the source - * topic, not historical) for the given capture token. Callers override the - * pieces a specific case exercises (event name, lane, source topic). - */ -const buildRecord = (overrides: RecordOverrides): Effect.Effect => - Effect.gen(function* () { - const lane = overrides.lane ?? "main"; - const sourceTopic = overrides.sourceTopic ?? `analytics.events.${lane}.v1`; - const captureId = overrides.captureId ?? (yield* unique("capture")); - const distinctId = overrides.distinctId ?? (yield* unique("distinct")); - const isHistorical = lane === "historical"; - - const capturedEvent: CapturedEventV1Type = { - schemaVersion: 1, - captureId, - token: overrides.token, - organizationId, - projectId, - event: overrides.event ?? "page_view", - distinctId, - eventTimestamp: "2020-01-01T00:00:00.000Z", - receivedAt: "2020-01-01T00:00:01.000Z", - properties: overrides.properties ?? {}, - context: {}, - rawPayload: {}, - request: { requestId: yield* unique("req") }, - routing: { - routeClass: routeClassOf(lane), - targetTopic: sourceTopic, - isHistorical, - skipEnrichment: false, - }, - }; - - return { - capturedEvent, - headers: {}, - lane, - rawValue: encodeJson(capturedEvent), - sourceOffset: `${sourceTopic}:0:1`, - sourcePartition: 0, - sourceTopic, - }; - }); - -/** Insert a public API key (the capture token) under the fixture project. */ -const insertPublicApiKey = (token: string) => - Effect.gen(function* () { - const db = yield* Db; - const id = yield* unique("apikey"); - yield* db.insert(apiKeys).values({ - end: token.slice(-4), - id, - isPublic: true, - key: token, - name: "Integration capture key", - prefix: "it", - projectId, - }); - return id; - }); - -/** Insert a processor policy row for the fixture project with the given fields. */ -const insertPolicy = (overrides: { - readonly processorEnabled?: boolean; - readonly processorAllowOverflow?: boolean; - readonly processorAllowHistorical?: boolean; -}) => - Effect.gen(function* () { - const db = yield* Db; - yield* db - .insert(captureProjectPolicies) - .values({ - processorAllowHistorical: overrides.processorAllowHistorical ?? true, - processorAllowOverflow: overrides.processorAllowOverflow ?? true, - processorEnabled: overrides.processorEnabled ?? true, - projectId, - }) - .onConflictDoUpdate({ - target: captureProjectPolicies.projectId, - set: { - processorAllowHistorical: overrides.processorAllowHistorical ?? true, - processorAllowOverflow: overrides.processorAllowOverflow ?? true, - processorEnabled: overrides.processorEnabled ?? true, - }, - }); - }); - -/** Read the DLQ row recorded for a given capture id, bypassing the service. */ -const findDlqRowByCaptureId = (captureId: string) => - Effect.gen(function* () { - const db = yield* Db; - return yield* db.query.analyticsIngestDlq.findFirst({ - where: { captureId }, - }); - }); - -/** Read the identity mapping row for a distinct id under the fixture project. */ -const findIdentityRow = (distinctId: string) => - Effect.gen(function* () { - const db = yield* Db; - return yield* db.query.personIdentities.findFirst({ - where: { projectId, distinctId }, - }); - }); - -/** - * Delete everything the tests can create: DLQ rows by capture id, identity / - * personless / migration-job / person rows by id, policy + api-key rows. Each - * delete is `ignore`d so a missing row never turns the finalizer into a - * failure. - */ -interface Tracked { - readonly apiKeyIds: string[]; - readonly captureIds: string[]; - readonly distinctIds: string[]; - readonly clearedPolicy: { value: boolean }; -} - -const cleanup = (tracked: Tracked) => - Effect.gen(function* () { - const db = yield* Db; - - if (tracked.captureIds.length > 0) { - yield* db - .delete(analyticsIngestDlq) - .where(inArray(analyticsIngestDlq.captureId, tracked.captureIds)) - .pipe(Effect.ignore); - } - - if (tracked.distinctIds.length > 0) { - // Collect the person ids reachable from the created identity rows before - // deleting the mappings, so the parent `person` rows can be removed too. - const personRows = yield* db - .select({ personId: personIdentities.personId }) - .from(personIdentities) - .where( - and( - eq(personIdentities.projectId, projectId), - inArray(personIdentities.distinctId, tracked.distinctIds), - ), - ) - .pipe(Effect.orElseSucceed(() => NO_PERSON_ROWS)); - const personIds = [...new Set(personRows.map((row) => row.personId))]; - - yield* db - .delete(personIdentities) - .where( - and( - eq(personIdentities.projectId, projectId), - inArray(personIdentities.distinctId, tracked.distinctIds), - ), - ) - .pipe(Effect.ignore); - yield* db - .delete(personPersonlessIdentities) - .where( - and( - eq(personPersonlessIdentities.projectId, projectId), - inArray(personPersonlessIdentities.distinctId, tracked.distinctIds), - ), - ) - .pipe(Effect.ignore); - if (personIds.length > 0) { - yield* db.delete(persons).where(inArray(persons.id, personIds)).pipe(Effect.ignore); - } - } - - if (tracked.apiKeyIds.length > 0) { - yield* db.delete(apiKeys).where(inArray(apiKeys.id, tracked.apiKeyIds)).pipe(Effect.ignore); - } - - if (tracked.clearedPolicy.value) { - yield* db - .delete(captureProjectPolicies) - .where(eq(captureProjectPolicies.projectId, projectId)) - .pipe(Effect.ignore); - } - }); - -/** - * Wrap a test body so everything it creates is removed afterward, regardless of - * how the test exits. The body records ids into the shared {@link Tracked} bag - * via the helpers it is handed; cleanup reads the collected ids lazily at - * finalization via `Effect.ensuring`, so it sees every id tracked while the - * body ran (including on failure). - */ -const withCleanup = ( - body: (track: { - readonly apiKey: (id: string) => void; - readonly capture: (id: string) => void; - readonly distinct: (id: string) => void; - readonly policy: () => void; - }) => Effect.Effect, -): Effect.Effect => { - const tracked: Tracked = { - apiKeyIds: [], - captureIds: [], - distinctIds: [], - clearedPolicy: { value: false }, - }; - return body({ - apiKey: (id) => tracked.apiKeyIds.push(id), - capture: (id) => tracked.captureIds.push(id), - distinct: (id) => tracked.distinctIds.push(id), - policy: () => { - tracked.clearedPolicy.value = true; - }, - }).pipe(Effect.ensuring(cleanup(tracked))); -}; - -describe("EventProcessorService.processRecordToOutputs", () => { - test( - "publishes a project_not_found DLQ row and returns empty outputs for an unknown token", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const captureId = yield* unique("capture-unknown"); - track.capture(captureId); - const record = yield* buildRecord({ captureId, token: yield* unique("missing-token") }); - - const outputs = yield* service.processRecordToOutputs(record); - - expect(outputs.processedEvents).toEqual([]); - expect(outputs.personEvents).toEqual([]); - expect(outputs.personIdentityEvents).toEqual([]); - - const dlqRow = yield* findDlqRowByCaptureId(captureId); - expect(dlqRow).toBeDefined(); - expect(dlqRow?.failureClass).toBe("project_not_found"); - // No resolved project, so the producer falls back to the captured event's project id. - expect(dlqRow?.projectId).toBe(projectId); - - // Nothing was identified. - expect(yield* findIdentityRow(record.capturedEvent.distinctId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "applies the default processor policy when no policy row exists and resolves identity", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-default"); - track.apiKey(yield* insertPublicApiKey(token)); - - const captureId = yield* unique("capture-default"); - const distinctId = yield* unique("distinct-default"); - track.capture(captureId); - track.distinct(distinctId); - const record = yield* buildRecord({ captureId, distinctId, token }); - - const outputs = yield* service.processRecordToOutputs(record); - - // Default policy enables the processor, so the record is processed. - expect(outputs.processedEvents.length).toBe(1); - const processed = outputs.processedEvents[0]!; - expect(processed.captureId).toBe(captureId); - expect(processed.projectId).toBe(projectId); - expect(processed.organizationId).toBe(organizationId); - expect(processed.identity.distinctId).toBe(distinctId); - - // No DLQ row was written on the happy path. - expect(yield* findDlqRowByCaptureId(captureId)).toBeUndefined(); - - // A non-anonymous distinct id creates a canonical person + identity mapping. - const identity = yield* findIdentityRow(distinctId); - expect(identity).toBeDefined(); - expect(identity?.personId).toBe(processed.identity.personId); - expect(outputs.personEvents.some((event) => event.personId === identity?.personId)).toBe( - true, - ); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "publishes a policy_rejected DLQ row and writes nothing when the processor is disabled", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-disabled"); - track.apiKey(yield* insertPublicApiKey(token)); - track.policy(); - yield* insertPolicy({ processorEnabled: false }); - - const captureId = yield* unique("capture-disabled"); - const distinctId = yield* unique("distinct-disabled"); - track.capture(captureId); - track.distinct(distinctId); - const record = yield* buildRecord({ captureId, distinctId, token }); - - const outputs = yield* service.processRecordToOutputs(record); - - expect(outputs.processedEvents).toEqual([]); - expect(outputs.personEvents).toEqual([]); - expect(outputs.personIdentityEvents).toEqual([]); - - const dlqRow = yield* findDlqRowByCaptureId(captureId); - expect(dlqRow).toBeDefined(); - expect(dlqRow?.failureClass).toBe("policy_rejected"); - expect(dlqRow?.projectId).toBe(projectId); - - // Rejected before identity resolution: no person mapping written. - expect(yield* findIdentityRow(distinctId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "publishes a schema_rejected DLQ row when routing fails the built-in validation rules", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-schema"); - track.apiKey(yield* insertPublicApiKey(token)); - - const captureId = yield* unique("capture-schema"); - const distinctId = yield* unique("distinct-schema"); - track.capture(captureId); - track.distinct(distinctId); - // targetTopic deliberately mismatches sourceTopic → validation rejects. - const record = yield* buildRecord({ captureId, distinctId, token }); - const mismatched: CapturedTransportRecord = { - ...record, - capturedEvent: { - ...record.capturedEvent, - routing: { ...record.capturedEvent.routing, targetTopic: "some.other.topic.v1" }, - }, - }; - - const outputs = yield* service.processRecordToOutputs(mismatched); - - expect(outputs.processedEvents).toEqual([]); - - const dlqRow = yield* findDlqRowByCaptureId(captureId); - expect(dlqRow).toBeDefined(); - expect(dlqRow?.failureClass).toBe("schema_rejected"); - expect(dlqRow?.projectId).toBe(projectId); - - expect(yield* findIdentityRow(distinctId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "publishes a reserved_event_name DLQ row for a reserved revenue event from an untrusted topic", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-reserved"); - track.apiKey(yield* insertPublicApiKey(token)); - - const captureId = yield* unique("capture-reserved"); - const distinctId = yield* unique("distinct-reserved"); - track.capture(captureId); - track.distinct(distinctId); - // A reserved $purchase.* event riding the normal (untrusted) main topic. - const record = yield* buildRecord({ - captureId, - distinctId, - token, - event: "$purchase.completed", - }); - - const outputs = yield* service.processRecordToOutputs(record); - - expect(outputs.processedEvents).toEqual([]); - - const dlqRow = yield* findDlqRowByCaptureId(captureId); - expect(dlqRow).toBeDefined(); - expect(dlqRow?.failureClass).toBe("reserved_event_name"); - expect(dlqRow?.failureMessage).toContain("$purchase.completed"); - expect(dlqRow?.projectId).toBe(projectId); - - expect(yield* findIdentityRow(distinctId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "allows a reserved revenue event when it arrives on the trusted source topic", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-trusted"); - track.apiKey(yield* insertPublicApiKey(token)); - - const captureId = yield* unique("capture-trusted"); - const distinctId = yield* unique("distinct-trusted"); - track.capture(captureId); - track.distinct(distinctId); - const record = yield* buildRecord({ - captureId, - distinctId, - token, - event: "$purchase.completed", - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }); - - const outputs = yield* service.processRecordToOutputs(record); - - // Trusted topic bypasses the reserved-name guard, so the event processes. - expect(outputs.processedEvents.length).toBe(1); - expect(outputs.processedEvents[0]!.event).toBe("$purchase.completed"); - expect(yield* findDlqRowByCaptureId(captureId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "honours a Resolved identity claim on the trusted topic — passes identity through and writes NO person/identity rows", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-resolved"); - track.apiKey(yield* insertPublicApiKey(token)); - - const captureId = yield* unique("capture-resolved"); - const distinctId = yield* unique("distinct-resolved"); - const personId = yield* unique("person-resolved"); - const clientEventId = yield* unique("evt-resolved"); - track.capture(captureId); - track.distinct(distinctId); - - // A trusted revenue event carrying a pre-resolved identity claim — the - // shape `dispatchTrusted` produces. The distinctId is non-anonymous, so - // the NORMAL path would create a person + identity row; the Resolved - // branch must skip all of that (the "revenue writes no person rows" - // characterization). - const base = yield* buildRecord({ - captureId, - distinctId, - token, - event: "$purchase.completed", - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }); - const record: CapturedTransportRecord = { - ...base, - capturedEvent: { - ...base.capturedEvent, - clientEventId, - identityClaim: { _tag: "Resolved", distinctId, personId }, - trustClass: "trusted-revenue", - routing: { ...base.capturedEvent.routing, skipEnrichment: true }, - }, - }; - - const outputs = yield* service.processRecordToOutputs(record); - - // The processed event carries the claimed identity verbatim, and the - // deterministic clientEventId becomes the event_id (processedEventId). - expect(outputs.processedEvents.length).toBe(1); - const processed = outputs.processedEvents[0]!; - expect(processed.event).toBe("$purchase.completed"); - expect(processed.identity).toEqual({ distinctId, mode: "full", personId }); - expect(processed.processedEventId).toBe(clientEventId); - - // No person/identity wire events emitted... - expect(outputs.personEvents).toEqual([]); - expect(outputs.personIdentityEvents).toEqual([]); - // ...and no identity row written to the DB (revenue writes no person rows). - expect(yield* findIdentityRow(distinctId)).toBeUndefined(); - expect(yield* findDlqRowByCaptureId(captureId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "resolves a trusted Resolved event by projectId even when its token has no api_key (synthetic server token)", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - // Deliberately NO api key inserted: revenue projects without a public - // SDK key thread a synthetic `vh_server_revenue_*` token. The trusted - // Resolved branch must resolve the project by its (server-stamped) - // projectId rather than DLQ on the missing token. - const syntheticToken = `vh_server_revenue_${projectId}`; - const captureId = yield* unique("capture-synthetic"); - const distinctId = yield* unique("distinct-synthetic"); - const personId = yield* unique("person-synthetic"); - track.capture(captureId); - track.distinct(distinctId); - - const base = yield* buildRecord({ - captureId, - distinctId, - token: syntheticToken, - event: "$subscription.renewed", - sourceTopic: REVENUE_TRUSTED_SOURCE_TOPIC, - }); - const record: CapturedTransportRecord = { - ...base, - capturedEvent: { - ...base.capturedEvent, - clientEventId: yield* unique("evt-synthetic"), - identityClaim: { _tag: "Resolved", distinctId, personId }, - trustClass: "trusted-revenue", - routing: { ...base.capturedEvent.routing, skipEnrichment: true }, - }, - }; - - const outputs = yield* service.processRecordToOutputs(record); - - expect(outputs.processedEvents.length).toBe(1); - expect(outputs.processedEvents[0]!.organizationId).toBe(organizationId); - expect(outputs.processedEvents[0]!.identity.personId).toBe(personId); - expect(outputs.personEvents).toEqual([]); - expect(yield* findDlqRowByCaptureId(captureId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "ignores a Resolved claim on an UNTRUSTED topic (defence-in-depth) and resolves identity normally", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-forged"); - track.apiKey(yield* insertPublicApiKey(token)); - - const captureId = yield* unique("capture-forged"); - const distinctId = yield* unique("distinct-forged"); - const forgedPersonId = yield* unique("person-forged"); - track.capture(captureId); - track.distinct(distinctId); - - // A non-reserved event on the NORMAL (untrusted) topic carrying a forged - // Resolved claim. The processor must NOT honour the claim — it falls - // through to real identity resolution, creating its own person. - const base = yield* buildRecord({ captureId, distinctId, token, event: "page_view" }); - const record: CapturedTransportRecord = { - ...base, - capturedEvent: { - ...base.capturedEvent, - identityClaim: { _tag: "Resolved", distinctId, personId: forgedPersonId }, - }, - }; - - const outputs = yield* service.processRecordToOutputs(record); - - expect(outputs.processedEvents.length).toBe(1); - // Real resolution ran: a person mapping exists and its id is NOT the - // forged one from the ignored claim. - const identity = yield* findIdentityRow(distinctId); - expect(identity).toBeDefined(); - expect(identity?.personId).not.toBe(forgedPersonId); - expect(outputs.processedEvents[0]!.identity.personId).toBe(identity?.personId); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "promotes an anonymous identity to the target person on an $identify event", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-identify"); - track.apiKey(yield* insertPublicApiKey(token)); - - const previousDistinctId = `vh:anon:${yield* unique("anon")}`; - const distinctId = yield* unique("identified"); - const captureId = yield* unique("capture-identify"); - track.capture(captureId); - track.distinct(distinctId); - track.distinct(previousDistinctId); - - const record = yield* buildRecord({ - captureId, - distinctId, - token, - event: "$identify", - properties: { - $set: { email: "person@voidhash.test" }, - $previous_distinct_id: previousDistinctId, - }, - }); - - const outputs = yield* service.processRecordToOutputs(record); - - expect(outputs.processedEvents.length).toBe(1); - const processed = outputs.processedEvents[0]!; - expect(processed.event).toBe("$identify"); - expect(processed.identity.distinctId).toBe(distinctId); - expect(processed.identity.mode).toBe("full"); - - // The identified distinct id is mapped to a real person. - const identity = yield* findIdentityRow(distinctId); - expect(identity).toBeDefined(); - expect(identity?.personId).toBe(processed.identity.personId); - - // The identify produced a person snapshot output for the target person. - expect(outputs.personEvents.some((event) => event.personId === identity?.personId)).toBe( - true, - ); - expect(yield* findDlqRowByCaptureId(captureId)).toBeUndefined(); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); - - test( - "serialises concurrent records that share the same (token, distinctId) identity key", - withCleanup((track) => - Effect.gen(function* () { - const service = yield* EventProcessorService; - - const token = yield* unique("token-serial"); - track.apiKey(yield* insertPublicApiKey(token)); - - // Same distinct id (and token) for both records → same identity key, so - // the scheduler must run their identity-resolution effects one at a - // time. A non-anonymous distinct id with shouldCreatePerson=true would - // race on the unique (project_id, distinct_id) identity index if the - // two effects overlapped; serialisation means both observe one person. - const distinctId = yield* unique("distinct-serial"); - track.distinct(distinctId); - const captureA = yield* unique("capture-serial-a"); - const captureB = yield* unique("capture-serial-b"); - track.capture(captureA); - track.capture(captureB); - - const recordA = yield* buildRecord({ captureId: captureA, distinctId, token }); - const recordB = yield* buildRecord({ captureId: captureB, distinctId, token }); - - const [outputsA, outputsB] = yield* Effect.all( - [service.processRecordToOutputs(recordA), service.processRecordToOutputs(recordB)], - { concurrency: 2 }, - ); - - expect(outputsA.processedEvents.length).toBe(1); - expect(outputsB.processedEvents.length).toBe(1); - - // Exactly one canonical person backs the shared distinct id. - const identity = yield* findIdentityRow(distinctId); - expect(identity).toBeDefined(); - expect(outputsA.processedEvents[0]!.identity.personId).toBe(identity?.personId); - expect(outputsB.processedEvents[0]!.identity.personId).toBe(identity?.personId); - - const db = yield* Db; - const identityRows = yield* db - .select({ id: personIdentities.id }) - .from(personIdentities) - .where( - and( - eq(personIdentities.projectId, projectId), - eq(personIdentities.distinctId, distinctId), - ), - ); - expect(identityRows.length).toBe(1); - }), - ).pipe(Effect.provide(TestLayer), CoreAuthSession.authenticate()), - ); -}); diff --git a/packages/core/test/services/analyticsIngest/EventProcessorService.test.ts b/packages/core/test/services/analyticsIngest/EventProcessorService.test.ts deleted file mode 100644 index 0e97c8255..000000000 --- a/packages/core/test/services/analyticsIngest/EventProcessorService.test.ts +++ /dev/null @@ -1,626 +0,0 @@ -import { DateTime, Effect, Schema } from "effect"; - -import { ANONYMOUS_USER_ID_PREFIX } from "@voidhash/lib"; -import { constant } from "@voidhash/lib/lang"; - -import type { - CapturedEventV1Type, - CapturedTransportRecord, - ProcessingEvent, - ProcessorProjectPolicy, - ResolvedProcessorProject, -} from "../../../src/domain/analyticsIngest/AnalyticsIngest.ts"; -import type { - PersonIdentityEventV1, - PersonSnapshotEventV1, -} from "../../../src/services/personIdentity/PersonIdentityService.ts"; -import { - attachProjectPolicy, - buildPersonIdentityCall, - buildProcessedEvent, - toProcessorPersonEvent, - toProcessorPersonIdentityEvents, -} from "../../../src/services/analyticsIngest/EventProcessorService.ts"; -import { describe, expect, it } from "../../../src/testing/effect-vitest.ts"; - -const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); - -const PROJECT_ID = "prj_test"; -const ORG_ID = "org_test"; -const TOKEN = "tok_test"; -const SOURCE_TOPIC = "analytics.main"; -const EVENT_TS = "2026-01-01T00:00:00.000Z"; - -/** Fresh captured-event builder — one object per test, no shared mutable state. */ -const capturedEvent = (overrides: Partial = {}): CapturedEventV1Type => ({ - schemaVersion: 1, - captureId: "cap_1", - token: TOKEN, - organizationId: ORG_ID, - projectId: PROJECT_ID, - event: "page_view", - distinctId: "user_1", - eventTimestamp: EVENT_TS, - receivedAt: EVENT_TS, - properties: {}, - context: {}, - rawPayload: {}, - request: { requestId: "req_1" }, - routing: { - routeClass: "main", - targetTopic: SOURCE_TOPIC, - isHistorical: false, - skipEnrichment: false, - }, - ...overrides, -}); - -/** Fresh transport-record builder wrapping a captured event. */ -const transportRecord = ( - overrides: Partial = {}, -): CapturedTransportRecord => ({ - capturedEvent: overrides.capturedEvent ?? capturedEvent(), - headers: {}, - lane: "main", - rawValue: encodeJson(overrides.capturedEvent ?? capturedEvent()), - sourceOffset: "off_1", - sourcePartition: 0, - sourceTopic: SOURCE_TOPIC, - ...overrides, -}); - -/** Fresh processor-project-policy builder; permissive by default. */ -const policy = (overrides: Partial = {}): ProcessorProjectPolicy => ({ - processorAllowHistorical: true, - processorAllowOverflow: true, - processorEnabled: true, - processorHistoricalMinAgeHours: 24, - processorPersonProcessingEnabled: true, - processorSchemaMode: "lenient", - ...overrides, -}); - -/** Fresh resolved-project builder matching the default captured-event projectId. */ -const resolvedProject = ( - overrides: Partial = {}, -): ResolvedProcessorProject => ({ - organizationId: ORG_ID, - projectId: PROJECT_ID, - policy: policy(overrides.policy), - ...overrides, -}); - -/** Fresh processing-event builder — drives buildPersonIdentityCall. */ -const processingEvent = (overrides: Partial = {}): ProcessingEvent => ({ - capturedEvent: overrides.capturedEvent ?? capturedEvent(), - headers: {}, - identityKey: `${TOKEN}:user_1`, - lane: "main", - projectPolicy: policy(overrides.projectPolicy), - rawValue: "{}", - sourceOffset: "off_1", - sourcePartition: 0, - sourceTopic: SOURCE_TOPIC, - ...overrides, -}); - -const NOW = DateTime.toDateUtc(DateTime.makeUnsafe("2026-01-01T00:00:00.000Z")); - -describe("attachProjectPolicy", () => { - it("returns ok:true with a ProcessingEvent when every validation passes", () => { - const record = transportRecord(); - const result = attachProjectPolicy({ now: NOW, record, resolvedProject: resolvedProject() }); - - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.value.capturedEvent).toBe(record.capturedEvent); - expect(result.value.lane).toBe("main"); - expect(result.value.sourceTopic).toBe(SOURCE_TOPIC); - expect(result.value.projectPolicy.processorEnabled).toBe(true); - } - }); - - it("sets identityKey as `${token}:${distinctId}` on the success value", () => { - const record = transportRecord({ - capturedEvent: capturedEvent({ distinctId: "user_99", token: "tok_abc" }), - }); - const result = attachProjectPolicy({ now: NOW, record, resolvedProject: resolvedProject() }); - - expect(result.ok).toBe(true); - if (result.ok) { - expect(result.value.identityKey).toBe("tok_abc:user_99"); - } - }); - - it("rejects with a project_not_found DLQ event when the resolved project id does not match", () => { - const record = transportRecord(); - const result = attachProjectPolicy({ - now: NOW, - record, - resolvedProject: resolvedProject({ projectId: "prj_other" }), - }); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.value.failureClass).toBe("project_not_found"); - expect(result.value.projectId).toBe(PROJECT_ID); - } - }); - - it("rejects with a policy_rejected DLQ event when the processor is disabled for the project", () => { - const record = transportRecord(); - const result = attachProjectPolicy({ - now: NOW, - record, - resolvedProject: resolvedProject({ policy: policy({ processorEnabled: false }) }), - }); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.value.failureClass).toBe("policy_rejected"); - expect(result.value.failureMessage).toContain("disabled"); - } - }); - - it("rejects the overflow lane when processorAllowOverflow is false", () => { - const record = transportRecord({ lane: "overflow" }); - const result = attachProjectPolicy({ - now: NOW, - record, - resolvedProject: resolvedProject({ policy: policy({ processorAllowOverflow: false }) }), - }); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.value.failureClass).toBe("policy_rejected"); - expect(result.value.failureMessage).toContain("overflow"); - } - }); - - it("allows the overflow lane when processorAllowOverflow is true", () => { - const record = transportRecord({ lane: "overflow" }); - const result = attachProjectPolicy({ - now: NOW, - record, - resolvedProject: resolvedProject({ policy: policy({ processorAllowOverflow: true }) }), - }); - - expect(result.ok).toBe(true); - }); - - it("rejects the historical lane when processorAllowHistorical is false", () => { - // A well-formed historical record requires isHistorical=true and an event old - // enough to clear the min-age gate, so the lane check is what trips first. - const oldTimestamp = DateTime.formatIso( - DateTime.makeUnsafe(NOW.getTime() - 48 * 60 * 60 * 1000), - ); - const record = transportRecord({ - capturedEvent: capturedEvent({ - eventTimestamp: oldTimestamp, - routing: { - routeClass: "historical", - targetTopic: SOURCE_TOPIC, - isHistorical: true, - skipEnrichment: false, - }, - }), - lane: "historical", - }); - const result = attachProjectPolicy({ - now: NOW, - record, - resolvedProject: resolvedProject({ policy: policy({ processorAllowHistorical: false }) }), - }); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.value.failureClass).toBe("policy_rejected"); - expect(result.value.failureMessage).toContain("historical"); - } - }); - - it("rejects with a schema_rejected DLQ event when validateBuiltInProcessorRules fails", () => { - // routing.targetTopic mismatching the source topic is the first built-in rule. - const record = transportRecord({ - capturedEvent: capturedEvent({ - routing: { - routeClass: "main", - targetTopic: "some.other.topic", - isHistorical: false, - skipEnrichment: false, - }, - }), - }); - const result = attachProjectPolicy({ now: NOW, record, resolvedProject: resolvedProject() }); - - expect(result.ok).toBe(false); - if (!result.ok) { - expect(result.value.failureClass).toBe("schema_rejected"); - } - }); -}); - -describe("buildPersonIdentityCall", () => { - it.effect("returns kind:'identify' when the event is $identify", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - event: "$identify", - distinctId: "identified_user", - properties: { $previous_distinct_id: "anon_123" }, - }), - }), - ); - - expect(call.kind).toBe("identify"); - if (call.kind === "identify") { - expect(call.input.distinctId).toBe("identified_user"); - expect(call.input.projectId).toBe(PROJECT_ID); - expect(call.input.eventTimestamp).toEqual( - DateTime.toDateUtc(DateTime.makeUnsafe(EVENT_TS)), - ); - } - }), - ); - - it.effect("includes previousDistinctId from $previous_distinct_id for $identify", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - event: "$identify", - properties: { $previous_distinct_id: "anon_prev" }, - }), - }), - ); - - expect(call.kind).toBe("identify"); - if (call.kind === "identify") { - expect(call.input.previousDistinctId).toBe("anon_prev"); - } - }), - ); - - it.effect("fails when $identify is missing $previous_distinct_id", () => - Effect.gen(function* () { - const error = yield* Effect.flip( - buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ event: "$identify", properties: {} }), - }), - ), - ); - - expect(error.message).toContain("$previous_distinct_id"); - }), - ); - - it.effect("returns kind:'resolve' for non-$identify events", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ capturedEvent: capturedEvent({ event: "page_view" }) }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.distinctId).toBe("user_1"); - expect(call.input.projectId).toBe(PROJECT_ID); - } - }), - ); - - it.effect("parses name/email and set/setOnce attributes from $set and $set_once", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - properties: { - $set: { name: "Ada", plan: "pro" }, - $set_once: { email: "ada@example.com", signup_source: "web" }, - }, - }), - }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.name).toBe("Ada"); - expect(call.input.email).toBe("ada@example.com"); - expect(call.input.setAttributes).toEqual({ plan: "pro" }); - expect(call.input.setOnceAttributes).toEqual({ signup_source: "web" }); - } - }), - ); - - it.effect("excludes name and email from setAttributes and setOnceAttributes", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - properties: { - $set: { name: "Ada", email: "a@b.co", plan: "pro" }, - $set_once: { name: "Once", email: "x@y.co", region: "eu" }, - }, - }), - }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.setAttributes).not.toHaveProperty("name"); - expect(call.input.setAttributes).not.toHaveProperty("email"); - expect(call.input.setOnceAttributes).not.toHaveProperty("name"); - expect(call.input.setOnceAttributes).not.toHaveProperty("email"); - expect(call.input.setAttributes).toEqual({ plan: "pro" }); - expect(call.input.setOnceAttributes).toEqual({ region: "eu" }); - } - }), - ); - - it.effect("reads traits from a nested `properties.properties` envelope", () => - Effect.gen(function* () { - // extractInnerProperties unwraps one level when properties.properties is a plain object. - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - properties: { properties: { $set: { name: "Inner" } } }, - }), - }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.name).toBe("Inner"); - } - }), - ); - - it.effect("defaults shouldCreatePerson to false for an anonymous distinct id", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ distinctId: `${ANONYMOUS_USER_ID_PREFIX}abc` }), - }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.shouldCreatePerson).toBe(false); - } - }), - ); - - it.effect("defaults shouldCreatePerson to true for a non-anonymous distinct id", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ capturedEvent: capturedEvent({ distinctId: "real_user" }) }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.shouldCreatePerson).toBe(true); - } - }), - ); - - it.effect("honors an explicit $process_person_profile over the distinct-id default", () => - Effect.gen(function* () { - // Anonymous id would default to false, but the explicit flag wins. - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - distinctId: `${ANONYMOUS_USER_ID_PREFIX}abc`, - properties: { $process_person_profile: true }, - }), - }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.shouldCreatePerson).toBe(true); - } - }), - ); - - it.effect("empties enrichment attributes when routing.skipEnrichment is true", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - properties: { $set: { plan: "pro" }, $set_once: { region: "eu" } }, - routing: { - routeClass: "main", - targetTopic: SOURCE_TOPIC, - isHistorical: false, - skipEnrichment: true, - }, - }), - }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.setAttributes).toEqual({}); - expect(call.input.setOnceAttributes).toEqual({}); - } - }), - ); - - it.effect("empties enrichment attributes when processorPersonProcessingEnabled is false", () => - Effect.gen(function* () { - const call = yield* buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ - properties: { $set: { plan: "pro" }, $set_once: { region: "eu" } }, - }), - projectPolicy: policy({ processorPersonProcessingEnabled: false }), - }), - ); - - expect(call.kind).toBe("resolve"); - if (call.kind === "resolve") { - expect(call.input.setAttributes).toEqual({}); - expect(call.input.setOnceAttributes).toEqual({}); - } - }), - ); - - it.effect("fails when $set is present but not an object (parsePersonTraits failure)", () => - Effect.gen(function* () { - const error = yield* Effect.flip( - buildPersonIdentityCall( - processingEvent({ - capturedEvent: capturedEvent({ properties: { $set: "not-an-object" } }), - }), - ), - ); - - expect(error.message).toContain("$set must be an object"); - }), - ); -}); - -describe("toProcessorPersonEvent", () => { - /** Fresh person-snapshot builder. */ - const snapshot = (overrides: Partial = {}): PersonSnapshotEventV1 => ({ - changedAt: EVENT_TS, - personId: "person_1", - isArchived: false, - projectId: PROJECT_ID, - schemaVersion: 1, - traits: { plan: "pro" }, - version: 3, - ...overrides, - }); - - it("maps all snapshot fields, including the optional ones when present", () => { - const event = toProcessorPersonEvent( - snapshot({ - email: "a@b.co", - mergedIntoPersonId: "person_2", - name: "Ada", - primaryDistinctId: "user_1", - }), - ); - - expect(event).toEqual({ - changedAt: EVENT_TS, - personId: "person_1", - email: "a@b.co", - isArchived: false, - mergedIntoPersonId: "person_2", - name: "Ada", - primaryDistinctId: "user_1", - projectId: PROJECT_ID, - schemaVersion: 1, - traits: { plan: "pro" }, - version: 3, - }); - }); - - it("omits email/mergedIntoPersonId/name/primaryDistinctId when absent", () => { - const event = toProcessorPersonEvent(snapshot()); - - expect(event).not.toHaveProperty("email"); - expect(event).not.toHaveProperty("mergedIntoPersonId"); - expect(event).not.toHaveProperty("name"); - expect(event).not.toHaveProperty("primaryDistinctId"); - expect(event.personId).toBe("person_1"); - expect(event.schemaVersion).toBe(1); - }); -}); - -describe("toProcessorPersonIdentityEvents", () => { - /** Fresh mapping-event builder. */ - const mappingEvent = (overrides: Partial = {}): PersonIdentityEventV1 => ({ - changedAt: EVENT_TS, - personId: "person_1", - distinctId: "user_1", - isDeleted: false, - kind: 2, - projectId: PROJECT_ID, - schemaVersion: 1, - version: 2, - ...overrides, - }); - - const identity = (distinctId: string) => ({ - distinctId, - mode: constant("full"), - personId: "person_1", - }); - - it("maps each mappingEvent to a ProcessorPersonIdentityEventV1", () => { - const events = toProcessorPersonIdentityEvents({ - identity: identity("user_1"), - mappingEvents: [mappingEvent(), mappingEvent({ distinctId: "user_1", version: 5 })], - }); - - expect(events).toHaveLength(2); - expect(events[0]?.personId).toBe("person_1"); - expect(events[0]?.projectId).toBe(PROJECT_ID); - expect(events[1]?.version).toBe(5); - }); - - it("returns an empty array when there are no mapping events", () => { - const events = toProcessorPersonIdentityEvents({ - identity: identity("user_1"), - mappingEvents: [], - }); - - expect(events).toEqual([]); - }); - - it("omits previousDistinctId when mappingEvent.distinctId equals the identity distinctId", () => { - const events = toProcessorPersonIdentityEvents({ - identity: identity("user_1"), - mappingEvents: [mappingEvent({ distinctId: "user_1" })], - }); - - expect(events[0]).not.toHaveProperty("previousDistinctId"); - expect(events[0]?.distinctId).toBe("user_1"); - }); - - it("sets previousDistinctId and rewrites distinctId when the mapping distinctId differs", () => { - // When the mapping points at a different source id, the source becomes - // previousDistinctId and the row's distinctId is the identity's canonical id. - const events = toProcessorPersonIdentityEvents({ - identity: identity("canonical_user"), - mappingEvents: [mappingEvent({ distinctId: "anon_source" })], - }); - - expect(events[0]?.previousDistinctId).toBe("anon_source"); - expect(events[0]?.distinctId).toBe("canonical_user"); - }); -}); - -describe("buildProcessedEvent", () => { - it("uses captureId as the deterministic processed event id when no client uuid", () => { - const event = capturedEvent(); - const processed = buildProcessedEvent({ - capturedEvent: event, - identity: { distinctId: event.distinctId, mode: "personless" }, - lane: "main", - sourceOffset: "0", - sourcePartition: 0, - sourceTopic: event.routing.targetTopic, - }); - - expect(processed.processedEventId).toBe(event.captureId); - }); - - it("prefers the SDK clientEventId over captureId for the processed event id", () => { - const event = capturedEvent({ clientEventId: "client_uuid" }); - const processed = buildProcessedEvent({ - capturedEvent: event, - identity: { distinctId: event.distinctId, mode: "personless" }, - lane: "main", - sourceOffset: "0", - sourcePartition: 0, - sourceTopic: event.routing.targetTopic, - }); - - expect(processed.processedEventId).toBe("client_uuid"); - }); -}); diff --git a/packages/core/test/services/apiKeys/ApiKeyService.integration.test.ts b/packages/core/test/services/apiKeys/ApiKeyService.integration.test.ts index ccd875302..8449b08b5 100644 --- a/packages/core/test/services/apiKeys/ApiKeyService.integration.test.ts +++ b/packages/core/test/services/apiKeys/ApiKeyService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link ApiKeyService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * and PostgreSQL; only the project schema cache is an in-memory stub). * * The service manages three kinds of keys, each with a distinct contract that * these tests exercise end-to-end and verify by reading the persisted row back: diff --git a/packages/core/test/services/featureFlags/FeatureFlagService.integration.test.ts b/packages/core/test/services/featureFlags/FeatureFlagService.integration.test.ts index e6300d132..708db2006 100644 --- a/packages/core/test/services/featureFlags/FeatureFlagService.integration.test.ts +++ b/packages/core/test/services/featureFlags/FeatureFlagService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link FeatureFlagService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB + - * ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * PostgreSQL; only the project schema cache is an in-memory stub). * * Each test drives a public method end-to-end and verifies the *persisted* side * effects, not just the return value: diff --git a/packages/core/test/services/paymentProviders/PaymentProviderConfigurationService.integration.test.ts b/packages/core/test/services/paymentProviders/PaymentProviderConfigurationService.integration.test.ts index ac157cfb1..8d0c06a94 100644 --- a/packages/core/test/services/paymentProviders/PaymentProviderConfigurationService.integration.test.ts +++ b/packages/core/test/services/paymentProviders/PaymentProviderConfigurationService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PaymentProviderConfigurationService}, run against * the real backend stack provisioned once by `test/_testing/globalSetup.ts` - * (live PlanetScale DB + ClickHouse + WorkOS; only the project schema cache is + * (live PostgreSQL; only the project schema cache is * an in-memory stub). * * The service orchestrates four collaborators — permission checks, the diff --git a/packages/core/test/services/paymentProviders/PaymentProviderProductService.integration.test.ts b/packages/core/test/services/paymentProviders/PaymentProviderProductService.integration.test.ts index 532381ccf..0fbad24e4 100644 --- a/packages/core/test/services/paymentProviders/PaymentProviderProductService.integration.test.ts +++ b/packages/core/test/services/paymentProviders/PaymentProviderProductService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PaymentProviderProductService}, run against the * real backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB + ClickHouse + WorkOS; only the project schema cache is an + * PostgreSQL; only the project schema cache is an * in-memory stub). * * The service maps catalog `product` rows to provider-specific product keys via diff --git a/packages/core/test/services/paymentProviders/appStore/AppStorePaymentProvider.integration.test.ts b/packages/core/test/services/paymentProviders/appStore/AppStorePaymentProvider.integration.test.ts index f57cc2aca..ccfe8157f 100644 --- a/packages/core/test/services/paymentProviders/appStore/AppStorePaymentProvider.integration.test.ts +++ b/packages/core/test/services/paymentProviders/appStore/AppStorePaymentProvider.integration.test.ts @@ -3,7 +3,7 @@ * ({@link AppStorePaymentProvider} — exported as `AppStorePaymentProviderEngine` * from `@voidhash/core/services`), run against the real backend stack * provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB + - * ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * PostgreSQL; only the project schema cache is an in-memory stub). * * The engine is a multi-branch purchase-recording machine: every `record*` * method first resolves the App Store identity (creating / reusing a person and diff --git a/packages/core/test/services/paymentProviders/appStore/AppStoreReconciliationService.integration.test.ts b/packages/core/test/services/paymentProviders/appStore/AppStoreReconciliationService.integration.test.ts index 69ce9347a..76c35d7a8 100644 --- a/packages/core/test/services/paymentProviders/appStore/AppStoreReconciliationService.integration.test.ts +++ b/packages/core/test/services/paymentProviders/appStore/AppStoreReconciliationService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link AppStoreReconciliationService}, run against the * real backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB + ClickHouse + WorkOS; only the project schema cache is an + * PostgreSQL; only the project schema cache is an * in-memory stub). * * `reconcileOriginalTransaction` is a heavy replay engine: it resolves the @@ -33,8 +33,8 @@ * Conventions: * - The collaborators the service-under-test layer needs but the harness does * not provide (`AppStorePaymentProvider` and its whole record graph) are - * wired here from their real layers, mirroring `AppStoreRuntimeLayers.ts` and - * the {@link EventProcessorService} integration test. The FX fetcher and the + * wired here from their real layers, mirroring `AppStoreRuntimeLayers.ts`. + * The FX fetcher and the * workflow/queue ports that carry no in-process seam are wired to stubs; * none of them is reached on the pre-network paths exercised here. * - Each test creates its own `payment_provider_configuration` row under the diff --git a/packages/core/test/services/paymentProviders/googlePlay/GooglePlayPaymentProvider.integration.test.ts b/packages/core/test/services/paymentProviders/googlePlay/GooglePlayPaymentProvider.integration.test.ts index 8b0dc2848..1f8591271 100644 --- a/packages/core/test/services/paymentProviders/googlePlay/GooglePlayPaymentProvider.integration.test.ts +++ b/packages/core/test/services/paymentProviders/googlePlay/GooglePlayPaymentProvider.integration.test.ts @@ -2,7 +2,7 @@ * Integration tests for the Google Play record engine * ({@link GooglePlayPaymentProvider}), run against the real backend stack * provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB + - * ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * PostgreSQL; only the project schema cache is an in-memory stub). * * The engine's `record*` methods consume an already-normalized purchase (the * Google analogue of App Store's decoded JWS), so these tests build synthetic diff --git a/packages/core/test/services/paymentProviders/stripe/StripePaymentProvider.integration.test.ts b/packages/core/test/services/paymentProviders/stripe/StripePaymentProvider.integration.test.ts index cd1b8bbd1..b1ce09c1d 100644 --- a/packages/core/test/services/paymentProviders/stripe/StripePaymentProvider.integration.test.ts +++ b/packages/core/test/services/paymentProviders/stripe/StripePaymentProvider.integration.test.ts @@ -3,7 +3,7 @@ * — `core/StripePaymentProvider`, exported from the barrel as * `StripePaymentProviderEngine`), run against the real backend stack * provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB + - * ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * PostgreSQL; only the project schema cache is an in-memory stub). * * The engine is a multi-branch purchase-recording machine: every `record*` * method decodes the verified Stripe `data.object`, resolves the Stripe diff --git a/packages/core/test/services/paymentProviders/stripe/StripeWebhookHandlerService.integration.test.ts b/packages/core/test/services/paymentProviders/stripe/StripeWebhookHandlerService.integration.test.ts index 73ed10e04..cef44e9ed 100644 --- a/packages/core/test/services/paymentProviders/stripe/StripeWebhookHandlerService.integration.test.ts +++ b/packages/core/test/services/paymentProviders/stripe/StripeWebhookHandlerService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link StripeWebhookHandlerService} — the Stripe webhook * ingress — run against the real backend stack provisioned once by - * `test/_testing/globalSetup.ts` (live PlanetScale DB + ClickHouse + WorkOS). + * `test/_testing/globalSetup.ts` (live PostgreSQL). * * Unlike the App Store webhook handler — whose every branch is gated behind a * real Apple-signed JWS chained to Apple's production/sandbox CA (no in-process diff --git a/packages/core/test/services/paywallLocations/PaywallLocationService.integration.test.ts b/packages/core/test/services/paywallLocations/PaywallLocationService.integration.test.ts index ed759e94e..68777f353 100644 --- a/packages/core/test/services/paywallLocations/PaywallLocationService.integration.test.ts +++ b/packages/core/test/services/paywallLocations/PaywallLocationService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PaywallLocationService}, run against the real * backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB + ClickHouse + WorkOS; only the project schema cache is an + * PostgreSQL; only the project schema cache is an * in-memory stub). * * Every test drives the service end-to-end and verifies the *persisted* side diff --git a/packages/core/test/services/paywalls/PaywallService.integration.test.ts b/packages/core/test/services/paywalls/PaywallService.integration.test.ts index f94355d77..c812c7ce1 100644 --- a/packages/core/test/services/paywalls/PaywallService.integration.test.ts +++ b/packages/core/test/services/paywalls/PaywallService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PaywallService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * and PostgreSQL; only the project schema cache is an in-memory stub). * * Each test drives the service end-to-end and verifies the *persisted* side * effects rather than just the method's return value: diff --git a/packages/core/test/services/perkGrants/PerkGrantService.integration.test.ts b/packages/core/test/services/perkGrants/PerkGrantService.integration.test.ts index 03c83e727..df4065766 100644 --- a/packages/core/test/services/perkGrants/PerkGrantService.integration.test.ts +++ b/packages/core/test/services/perkGrants/PerkGrantService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PerkGrantService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * and PostgreSQL; only the project schema cache is an in-memory stub). * * `PerkGrantService` reconciles a person's `person_unlocked_perk` rows against * their `subscription` / `purchase` rows and the product→perk catalog. There is diff --git a/packages/core/test/services/perks/PerkService.integration.test.ts b/packages/core/test/services/perks/PerkService.integration.test.ts index c3b701ce9..2323d1e01 100644 --- a/packages/core/test/services/perks/PerkService.integration.test.ts +++ b/packages/core/test/services/perks/PerkService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PerkService}, run against the real backend stack * provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB + - * ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * PostgreSQL; only the project schema cache is an in-memory stub). * * Each test drives the service end-to-end and verifies the *persisted* side * effects rather than just the method's return value: diff --git a/packages/core/test/services/personIdentity/IdentityProjectionPublisher.integration.test.ts b/packages/core/test/services/personIdentity/IdentityProjectionPublisher.integration.test.ts deleted file mode 100644 index 022ae5431..000000000 --- a/packages/core/test/services/personIdentity/IdentityProjectionPublisher.integration.test.ts +++ /dev/null @@ -1,574 +0,0 @@ -/** - * Integration tests for {@link IdentityProjectionPublisher}, run against the - * real backend stack provisioned once by `test/_testing/globalSetup.ts`. - * - * The unit under test is the `analyticsWriterLayer` composition: it transforms - * the `IdentityProjectionInput` (the person snapshots + identity mapping events - * the identify-completion workflow produces) into the analytics writer's tagged - * message union and hands them to the *real* {@link AnalyticsWriterService}, - * which fans them out into ClickHouse (`persons_v1` / `person_identity_v1`). - * - * Because the writer is real, these tests verify the *persisted* ClickHouse - * rows rather than the (discarded) intermediate `messageId`s — proving the two - * pure transforms (`toProcessorPersonEvent`, `toProcessorPersonIdentityEvent`) - * end-to-end: - * - person snapshots land in `persons_v1` with their traits / version and the - * organization id resolved from MySQL `project` by the writer, - * - identity mappings land in `person_identity_v1` with the correct - * `previous_distinct_id` derivation, - * - optional person fields (`email` / `name` / `primary_distinct_id` / - * `merged_into_person_id`) are omitted (written `NULL`) when absent and - * present when supplied, - * - a ClickHouse insert failure surfaces as the publisher's stable - * {@link QueueProducerError} tagged with `queueName: "AnalyticsWriterService"`. - * - * Conventions (mirroring {@link AnalyticsWriterService} integration tests): - * - The harness binds ClickHouse's read-write user, which reads cross-tenant - * rows with no row policy, so these tests both insert (via the publisher) and - * read their own rows back directly via {@link ClickhouseWebClient}. - * - Every row is namespaced by a unique `person_id` / `distinct_id` (via - * {@link uniqueId}) and asserted by membership, never by a table-wide count. - * - ClickHouse `MergeTree` has no cascading delete, so - * {@link withClickhouseCleanup} issues a best-effort `ALTER TABLE … DELETE` - * mutation per table on exit (success or failure) via `Effect.ensuring`. - * - Typed failures are asserted with `Effect.flip` + `instanceof`, paired with - * a state assertion on the failure path (project convention). - * - `publishIdentityResult` carries no `AuthSession` permission guard, so there - * is no forbidden-path case; the harness skeleton's - * `CoreAuthSession.authenticate()` is still applied for uniformity. - * - The `noop` layer is intentionally untested here: it returns `Effect.void` - * and is exercised through `PersonIdentityService`'s own tests (see the - * per-target plan notes). - */ -import { Clock, DateTime, Effect, Layer, Schema } from "effect"; -import { describe, expect } from "vitest"; - -import { IdentityProjectionPublisher } from "@voidhash/core/services/personIdentity/IdentityProjectionPublisher"; -import { AnalyticsWriterService } from "@voidhash/core/services/analyticsIngest/AnalyticsWriterService"; -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { QueueProducerError } from "@voidhash/core/services/infrastructure/QueueProducer"; -import type { - PersonIdentityEventV1, - PersonSnapshotEventV1, -} from "@voidhash/core/domain/person/Person"; - -import { CoreAuthSession } from "@testing/CoreAuthSession"; -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; -import { CoreTestFixture } from "@testing/CoreTestFixture"; - -const { test } = CoreIntegrationTestHarness.make(); - -const projectId = CoreTestFixture.projectId; -const organizationId = CoreTestFixture.organizationId; - -const PERSONS_TABLE = "persons_v1"; -const PERSON_IDENTITY_TABLE = "person_identity_v1"; -// A mapping event whose `distinctId` differs from `identity.distinctId` AND -// whose `version > 0` makes the writer's plan ALSO emit override + -// pending-override rows (keyed by the rewritten identity distinct id), so these -// must be reclaimed too — the global MySQL sweep never touches ClickHouse. -const PERSON_IDENTITY_OVERRIDES_TABLE = "person_identity_overrides_v1"; -const PERSON_IDENTITY_PENDING_OVERRIDES_TABLE = "person_identity_pending_overrides_v2"; - -/** - * The publisher's `analyticsWriterLayer` wired over the real - * {@link AnalyticsWriterService}. The writer in turn only needs `ClickhouseWebClient` - * and `Db`, both provided by the harness, so the test's remaining requirements stay - * within the harness service set. - */ -const PublisherLayer = IdentityProjectionPublisher.analyticsWriterLayer.pipe( - Layer.provide(AnalyticsWriterService.layer), -); - -/** Monotonic counter so ids stay unique even within the same millisecond. */ -let idSeq = 0; -const uniqueId = (label: string) => - Effect.map(Clock.currentTimeMillis, (now) => `it-ipp-${label}-${now}-${idSeq++}`); - -const decodeJson = Schema.decodeSync(Schema.UnknownFromJsonString); - -/** - * Carries only the optional person fields the caller actually supplied, so the - * omission branch of `toProcessorPersonEvent` stays reachable. - */ -const optionalPersonFields = (overrides: { - readonly email?: string; - readonly name?: string; - readonly mergedIntoPersonId?: string; - readonly primaryDistinctId?: string; -}): { - readonly email?: string; - readonly name?: string; - readonly mergedIntoPersonId?: string; - readonly primaryDistinctId?: string; -} => { - const fields: { - email?: string; - name?: string; - mergedIntoPersonId?: string; - primaryDistinctId?: string; - } = {}; - if (overrides.email) { - fields.email = overrides.email; - } - if (overrides.mergedIntoPersonId) { - fields.mergedIntoPersonId = overrides.mergedIntoPersonId; - } - if (overrides.name) { - fields.name = overrides.name; - } - if (overrides.primaryDistinctId) { - fields.primaryDistinctId = overrides.primaryDistinctId; - } - return fields; -}; - -/** - * Build a {@link PersonSnapshotEventV1}. Optional fields (`email` / `name` / - * `mergedIntoPersonId` / `primaryDistinctId`) are only set when supplied, so a - * test can drive the optional-field omission branch of `toProcessorPersonEvent`. - */ -const personEvent = (overrides: { - readonly personId: string; - readonly email?: string; - readonly name?: string; - readonly mergedIntoPersonId?: string; - readonly primaryDistinctId?: string; - readonly traits?: Record; - readonly version?: number; - readonly isArchived?: boolean; -}): Effect.Effect => - Effect.map(DateTime.nowAsDate, (now) => ({ - changedAt: now.toISOString(), - personId: overrides.personId, - isArchived: overrides.isArchived ?? false, - ...optionalPersonFields(overrides), - projectId, - schemaVersion: 1, - traits: overrides.traits ?? {}, - version: overrides.version ?? 1, - })); - -/** Build a {@link PersonIdentityEventV1} mapping `distinctId` → `personId`. */ -const mappingEvent = (overrides: { - readonly personId: string; - readonly distinctId: string; - readonly version?: number; - readonly isDeleted?: boolean; -}): Effect.Effect => - Effect.map(DateTime.nowAsDate, (now) => ({ - changedAt: now.toISOString(), - distinctId: overrides.distinctId, - isDeleted: overrides.isDeleted ?? false, - // `kind` is part of the wire event but the publisher's identity transform - // ignores it (it never reaches ClickHouse); `2` is `PersonIdentityKind.Identified`, - // kept here only so the fixture stays realistic. - kind: 2, - personId: overrides.personId, - projectId, - schemaVersion: 1, - version: overrides.version ?? 1, - })); - -/** Read back the person rows for the given person ids straight from ClickHouse. */ -const findPersonRows = (personIds: ReadonlyArray) => - Effect.gen(function* () { - if (personIds.length === 0) return []; - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - return yield* ch<{ - person_id: string; - organization_id: string; - project_id: string; - email: string | null; - name: string | null; - primary_distinct_id: string | null; - merged_into_person_id: string | null; - is_archived: number; - version: string; - traits: string; - }>`SELECT person_id, organization_id, project_id, email, name, primary_distinct_id, - merged_into_person_id, is_archived, toString(version) AS version, traits - FROM ${ch.literal(PERSONS_TABLE)} WHERE person_id IN ${ch.param("Array(String)", [...personIds])}`; - }); - -/** - * Read back the identity-mapping rows for the given distinct ids. - * - * `person_identity_v1` is `ReplacingMergeTree(version)` keyed on - * `(project_id, distinct_id)`, so reads use `FINAL` to collapse duplicate sort - * keys to the surviving (highest-`version`) row deterministically — without it - * the result would depend on background merge timing. - */ -const findIdentityRows = (distinctIds: ReadonlyArray) => - Effect.gen(function* () { - if (distinctIds.length === 0) return []; - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - return yield* ch<{ - person_id: string; - organization_id: string; - project_id: string; - distinct_id: string; - previous_distinct_id: string | null; - is_deleted: number; - version: string; - }>`SELECT person_id, organization_id, project_id, distinct_id, previous_distinct_id, - is_deleted, toString(version) AS version - FROM ${ch.literal(PERSON_IDENTITY_TABLE)} FINAL WHERE distinct_id IN ${ch.param("Array(String)", [...distinctIds])}`; - }); - -/** - * Best-effort ClickHouse reclamation. MergeTree mutations are asynchronous, so - * each `ALTER TABLE … DELETE` is fire-and-forget and `ignore`d — a failed or - * slow mutation must never turn the finalizer into a test failure. Persons are - * cleared by `person_id`; identity rows by `distinct_id`. The override table is - * keyed by the same (rewritten) `distinct_id` and the pending-override table by - * `target_distinct_id`, so the tracked identity distinct ids reclaim those too - * — a mapping with a derived `previous_distinct_id` and `version > 0` lands rows - * in all three identity tables. - */ -const cleanupClickhouse = (created: { - readonly personIds: ReadonlyArray; - readonly distinctIds: ReadonlyArray; -}) => - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - const personIds = [...created.personIds]; - const distinctIds = [...created.distinctIds]; - - if (personIds.length > 0) { - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(PERSONS_TABLE)} DELETE WHERE person_id IN ${ch.param("Array(String)", personIds)}`, - ) - .pipe(Effect.ignore); - } - if (distinctIds.length > 0) { - for (const table of [PERSON_IDENTITY_TABLE, PERSON_IDENTITY_OVERRIDES_TABLE]) { - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(table)} DELETE WHERE distinct_id IN ${ch.param("Array(String)", distinctIds)}`, - ) - .pipe(Effect.ignore); - } - yield* ch - .asCommand( - ch`ALTER TABLE ${ch.literal(PERSON_IDENTITY_PENDING_OVERRIDES_TABLE)} DELETE WHERE target_distinct_id IN ${ch.param("Array(String)", distinctIds)}`, - ) - .pipe(Effect.ignore); - } - }); - -/** - * Wrap a test body so every ClickHouse row it writes is reclaimed afterward, - * regardless of how the test exits. Pass each written `person_id` to - * `trackPerson` and each `distinct_id` to `trackDistinct`; cleanup reads the - * collected ids lazily at finalization via `Effect.ensuring`. - */ -const withClickhouseCleanup = ( - body: (track: { - readonly trackPerson: (id: string) => void; - readonly trackDistinct: (id: string) => void; - }) => Effect.Effect, -): Effect.Effect => { - const personIds: string[] = []; - const distinctIds: string[] = []; - return body({ - trackDistinct: (id) => { - distinctIds.push(id); - }, - trackPerson: (id) => { - personIds.push(id); - }, - }).pipe(Effect.ensuring(cleanupClickhouse({ distinctIds, personIds }))); -}; - -describe("IdentityProjectionPublisher.publishIdentityResult (analyticsWriterLayer)", () => { - test( - "transforms an identified person snapshot + mapping event into ClickHouse rows", - withClickhouseCleanup(({ trackDistinct, trackPerson }) => - Effect.gen(function* () { - const publisher = yield* IdentityProjectionPublisher; - - const personId = yield* uniqueId("happy-person"); - const distinctId = yield* uniqueId("happy-distinct"); - trackPerson(personId); - trackDistinct(distinctId); - - yield* publisher.publishIdentityResult({ - identity: { distinctId }, - mappingEvents: [yield* mappingEvent({ distinctId, personId, version: 2 })], - personEvents: [ - yield* personEvent({ - email: "happy@voidhash.test", - name: "Happy Person", - personId, - primaryDistinctId: distinctId, - traits: { tier: "gold" }, - version: 5, - }), - ], - }); - - const personRows = yield* findPersonRows([personId]); - const personRow = personRows.find((row) => row.person_id === personId); - expect(personRow).toBeDefined(); - expect(personRow?.project_id).toBe(projectId); - // Writer resolves the organization from MySQL `project`, not from the event. - expect(personRow?.organization_id).toBe(organizationId); - expect(personRow?.email).toBe("happy@voidhash.test"); - expect(personRow?.name).toBe("Happy Person"); - expect(personRow?.primary_distinct_id).toBe(distinctId); - expect(personRow?.is_archived).toBe(0); - expect(personRow?.version).toBe("5"); - expect(decodeJson(personRow?.traits ?? "{}")).toEqual({ tier: "gold" }); - - const identityRows = yield* findIdentityRows([distinctId]); - const identityRow = identityRows.find((row) => row.distinct_id === distinctId); - expect(identityRow).toBeDefined(); - expect(identityRow?.person_id).toBe(personId); - expect(identityRow?.project_id).toBe(projectId); - expect(identityRow?.organization_id).toBe(organizationId); - expect(identityRow?.is_deleted).toBe(0); - expect(identityRow?.version).toBe("2"); - // distinctId === identity.distinctId → no previous distinct id derived. - expect(identityRow?.previous_distinct_id).toBeNull(); - }), - ).pipe(Effect.provide(PublisherLayer), CoreAuthSession.authenticate()), - ); - - test( - "omits optional person fields (email/name/primaryDistinctId/mergedIntoPersonId) when absent", - withClickhouseCleanup(({ trackPerson }) => - Effect.gen(function* () { - const publisher = yield* IdentityProjectionPublisher; - - const personId = yield* uniqueId("minimal-person"); - trackPerson(personId); - - // Only the required fields — every optional field is absent, so the - // transform must omit it (and the writer writes NULL). - yield* publisher.publishIdentityResult({ - identity: { distinctId: yield* uniqueId("minimal-distinct") }, - mappingEvents: [], - personEvents: [yield* personEvent({ personId })], - }); - - const personRows = yield* findPersonRows([personId]); - const personRow = personRows.find((row) => row.person_id === personId); - expect(personRow).toBeDefined(); - expect(personRow?.email).toBeNull(); - expect(personRow?.name).toBeNull(); - expect(personRow?.primary_distinct_id).toBeNull(); - expect(personRow?.merged_into_person_id).toBeNull(); - }), - ).pipe(Effect.provide(PublisherLayer), CoreAuthSession.authenticate()), - ); - - test( - "includes optional person fields (email/name/primaryDistinctId/mergedIntoPersonId) when present", - withClickhouseCleanup(({ trackPerson }) => - Effect.gen(function* () { - const publisher = yield* IdentityProjectionPublisher; - - const personId = yield* uniqueId("full-person"); - const mergedInto = yield* uniqueId("full-merged"); - const primaryDistinct = yield* uniqueId("full-primary"); - trackPerson(personId); - - yield* publisher.publishIdentityResult({ - identity: { distinctId: primaryDistinct }, - mappingEvents: [], - personEvents: [ - yield* personEvent({ - email: "full@voidhash.test", - isArchived: true, - mergedIntoPersonId: mergedInto, - name: "Full Person", - personId, - primaryDistinctId: primaryDistinct, - }), - ], - }); - - const personRows = yield* findPersonRows([personId]); - const personRow = personRows.find((row) => row.person_id === personId); - expect(personRow).toBeDefined(); - expect(personRow?.email).toBe("full@voidhash.test"); - expect(personRow?.name).toBe("Full Person"); - expect(personRow?.primary_distinct_id).toBe(primaryDistinct); - expect(personRow?.merged_into_person_id).toBe(mergedInto); - expect(personRow?.is_archived).toBe(1); - }), - ).pipe(Effect.provide(PublisherLayer), CoreAuthSession.authenticate()), - ); - - test( - "derives previous_distinct_id when the mapping distinct id differs from the identity distinct id", - withClickhouseCleanup(({ trackDistinct }) => - Effect.gen(function* () { - const publisher = yield* IdentityProjectionPublisher; - - const personId = yield* uniqueId("prev-person"); - const identityDistinct = yield* uniqueId("prev-identity"); - const mappingDistinct = yield* uniqueId("prev-mapping"); - // The persisted identity row is keyed by the identity distinct id (the - // transform rewrites `distinctId` to it and stores the mapping distinct - // id as `previous_distinct_id`), so reclaim by that id. Because a - // derived previous distinct id + `version > 0` also lands override + - // pending-override rows (both keyed by this same identity distinct id), - // tracking it reclaims all three identity tables. - trackDistinct(identityDistinct); - - yield* publisher.publishIdentityResult({ - identity: { distinctId: identityDistinct }, - mappingEvents: [yield* mappingEvent({ distinctId: mappingDistinct, personId, version: 3 })], - personEvents: [], - }); - - const identityRows = yield* findIdentityRows([identityDistinct]); - const identityRow = identityRows.find((row) => row.distinct_id === identityDistinct); - expect(identityRow).toBeDefined(); - expect(identityRow?.person_id).toBe(personId); - // distinctId rewritten to the identity distinct id, mapping distinct id - // becomes the previous distinct id. - expect(identityRow?.distinct_id).toBe(identityDistinct); - expect(identityRow?.previous_distinct_id).toBe(mappingDistinct); - expect(identityRow?.version).toBe("3"); - - // The mapping distinct id is NOT written as its own identity row. - const mappingRows = yield* findIdentityRows([mappingDistinct]); - expect(mappingRows.some((row) => row.distinct_id === mappingDistinct)).toBe(false); - }), - ).pipe(Effect.provide(PublisherLayer), CoreAuthSession.authenticate()), - ); - - test( - "omits previous_distinct_id when the mapping distinct id equals the identity distinct id", - withClickhouseCleanup(({ trackDistinct }) => - Effect.gen(function* () { - const publisher = yield* IdentityProjectionPublisher; - - const personId = yield* uniqueId("same-person"); - const distinctId = yield* uniqueId("same-distinct"); - trackDistinct(distinctId); - - yield* publisher.publishIdentityResult({ - identity: { distinctId }, - mappingEvents: [yield* mappingEvent({ distinctId, personId })], - personEvents: [], - }); - - const identityRows = yield* findIdentityRows([distinctId]); - const identityRow = identityRows.find((row) => row.distinct_id === distinctId); - expect(identityRow).toBeDefined(); - expect(identityRow?.distinct_id).toBe(distinctId); - expect(identityRow?.previous_distinct_id).toBeNull(); - }), - ).pipe(Effect.provide(PublisherLayer), CoreAuthSession.authenticate()), - ); - - test( - "writes multiple persons and identities in one batch", - withClickhouseCleanup(({ trackDistinct, trackPerson }) => - Effect.gen(function* () { - const publisher = yield* IdentityProjectionPublisher; - - const personA = yield* uniqueId("batch-person-a"); - const personB = yield* uniqueId("batch-person-b"); - const distinctA = yield* uniqueId("batch-distinct-a"); - const distinctB = yield* uniqueId("batch-distinct-b"); - trackPerson(personA); - trackPerson(personB); - // A single `publishIdentityResult` call carries ONE identity distinct id; - // `toProcessorPersonIdentityEvent` rewrites every mapping whose distinctId - // differs from it to that identity distinct id (storing the original as - // `previous_distinct_id`). With `identity.distinctId = distinctA`, A's - // mapping is kept as-is but B's mapping is rewritten to distinctA — so the - // only persisted identity `distinct_id` is distinctA, and that is the only - // id worth reclaiming (distinctB becomes a `previous_distinct_id`). - trackDistinct(distinctA); - - yield* publisher.publishIdentityResult({ - identity: { distinctId: distinctA }, - mappingEvents: [ - // distinctId === identity.distinctId → kept, no previous distinct id, - // version defaults to 1. - yield* mappingEvent({ distinctId: distinctA, personId: personA }), - // distinctId !== identity.distinctId → rewritten to distinctA with - // previous_distinct_id = distinctB (and version > 0 also lands - // override + pending-override rows). version = 2. - yield* mappingEvent({ distinctId: distinctB, personId: personB, version: 2 }), - ], - personEvents: [ - yield* personEvent({ name: "Batch A", personId: personA }), - yield* personEvent({ name: "Batch B", personId: personB }), - ], - }); - - // Person rows are keyed by (project_id, person_id), so the two persons - // land as independent rows regardless of the identity distinct id. - const personRows = yield* findPersonRows([personA, personB]); - expect(personRows.some((row) => row.person_id === personA)).toBe(true); - expect(personRows.some((row) => row.person_id === personB)).toBe(true); - - // BOTH mappings are rewritten/kept to the SAME identity distinct_id - // (distinctA), so they collide on the `person_identity_v1` sort key - // `(project_id, distinct_id)`. The table is `ReplacingMergeTree(version)`, - // so the two rows do NOT coexist — `FINAL` collapses them to the single - // surviving row with the highest `version`. A's mapping is version 1 and - // B's (rewritten) mapping is version 2, so the converged identity row is - // B's: person_id = personB, previous_distinct_id = distinctB, version 2. - // A's row is replaced and does not survive. - const identityRows = yield* findIdentityRows([distinctA]); - const survivor = identityRows.find((row) => row.distinct_id === distinctA); - expect(survivor).toBeDefined(); - expect(survivor?.person_id).toBe(personB); - expect(survivor?.previous_distinct_id).toBe(distinctB); - expect(survivor?.version).toBe("2"); - // The lower-version mapping (personA, version 1) loses the dedup and is - // not the surviving row under distinctA. - expect( - identityRows.some((row) => row.distinct_id === distinctA && row.person_id === personA), - ).toBe(false); - - // distinctB is never written as its own identity row (only as a - // previous distinct id), so a lookup by it returns nothing. - const distinctBRows = yield* findIdentityRows([distinctB]); - expect(distinctBRows.some((row) => row.distinct_id === distinctB)).toBe(false); - }), - ).pipe(Effect.provide(PublisherLayer), CoreAuthSession.authenticate()), - ); - - test( - "wraps a writer/ClickHouse insert failure as QueueProducerError(queueName=AnalyticsWriterService) and writes nothing", - // No cleanup wrapper: the rejected batch lands no rows. - Effect.gen(function* () { - const publisher = yield* IdentityProjectionPublisher; - - // `version` maps to the `UInt64` column `version`; a negative value is - // rejected by ClickHouse at insert time, surfacing as a SqlError - // the writer maps to AnalyticsWriterServiceError, which the publisher in - // turn wraps as its stable QueueProducerError. `changedAt` stays a valid - // ISO string so `toClickhouseTimestamp` does not throw first (that would - // be an uncaught defect, not the typed failure under test). - const personId = yield* uniqueId("bad-person"); - const error = yield* Effect.flip( - publisher.publishIdentityResult({ - identity: { distinctId: yield* uniqueId("bad-distinct") }, - mappingEvents: [], - personEvents: [yield* personEvent({ personId, version: -1 })], - }), - ); - - expect(error).toBeInstanceOf(QueueProducerError); - if (error instanceof QueueProducerError) { - expect(error._tag).toBe("QueueProducerError"); - expect(error.queueName).toBe("AnalyticsWriterService"); - } - - // The rejected person row must not have landed. - const personRows = yield* findPersonRows([personId]); - expect(personRows.some((row) => row.person_id === personId)).toBe(false); - }).pipe(Effect.provide(PublisherLayer), CoreAuthSession.authenticate()), - ); -}); diff --git a/packages/core/test/services/personIdentity/PersonIdentityService.integration.test.ts b/packages/core/test/services/personIdentity/PersonIdentityService.integration.test.ts index f1a604e60..46031c09f 100644 --- a/packages/core/test/services/personIdentity/PersonIdentityService.integration.test.ts +++ b/packages/core/test/services/personIdentity/PersonIdentityService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PersonIdentityService}, run against the real * backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB + ClickHouse + WorkOS; only the project schema cache is an + * PostgreSQL; only the project schema cache is an * in-memory stub). * * Both public methods (`resolveDistinctId` / `identifyDistinctId`) drive the @@ -16,7 +16,7 @@ * * Conventions: * - {@link IdentityProjectionPublisher.noop} replaces the analytics-writer - * publisher so the tests need no ClickHouse writer plumbing; a per-test + * publisher so the tests need no analytics-writer plumbing; a per-test * {@link trackingWorkflowLayer} stub captures the fire-and-forget workflow * dispatches so we can assert them deterministically. The full * service-under-test layer is provided at the pipe level so each test's diff --git a/packages/core/test/services/persons/PersonService.integration.test.ts b/packages/core/test/services/persons/PersonService.integration.test.ts index 86d9904f6..3500a4aa2 100644 --- a/packages/core/test/services/persons/PersonService.integration.test.ts +++ b/packages/core/test/services/persons/PersonService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PersonService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * and PostgreSQL; only the project schema cache is an in-memory stub). * * `PersonService` is the dashboard / admin surface for querying persons. It is * a *read-heavy* service: the only writes are `createPerson` (delegated to diff --git a/packages/core/test/services/productPerks/ProductPerkService.integration.test.ts b/packages/core/test/services/productPerks/ProductPerkService.integration.test.ts index fa118efd9..e1867cb5a 100644 --- a/packages/core/test/services/productPerks/ProductPerkService.integration.test.ts +++ b/packages/core/test/services/productPerks/ProductPerkService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link ProductPerkService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB + - * ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * PostgreSQL; only the project schema cache is an in-memory stub). * * The service owns the `(product, perk)` join-table aggregate. Each test drives * a public method end-to-end and verifies the *persisted* side effects, not just diff --git a/packages/core/test/services/products/ProductService.integration.test.ts b/packages/core/test/services/products/ProductService.integration.test.ts index 6237f4e05..39c934b63 100644 --- a/packages/core/test/services/products/ProductService.integration.test.ts +++ b/packages/core/test/services/products/ProductService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link ProductService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * and PostgreSQL; only the project schema cache is an in-memory stub). * * Each test drives the service end-to-end and verifies the *persisted* side * effects rather than just the method's return value: diff --git a/packages/core/test/services/projects/ProjectService.integration.test.ts b/packages/core/test/services/projects/ProjectService.integration.test.ts index 3e28453f9..31737000d 100644 --- a/packages/core/test/services/projects/ProjectService.integration.test.ts +++ b/packages/core/test/services/projects/ProjectService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link ProjectService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * and PostgreSQL; only the project schema cache is an in-memory stub). * * Each test drives the service end-to-end and verifies the *persisted* side * effects rather than just the method's return value: diff --git a/packages/core/test/services/purchaseProcessing/PurchaseLedgerWorkerService.integration.test.ts b/packages/core/test/services/purchaseProcessing/PurchaseLedgerWorkerService.integration.test.ts index 1ba36b122..2a5725286 100644 --- a/packages/core/test/services/purchaseProcessing/PurchaseLedgerWorkerService.integration.test.ts +++ b/packages/core/test/services/purchaseProcessing/PurchaseLedgerWorkerService.integration.test.ts @@ -102,7 +102,6 @@ const pollOptions = constant({ */ const makeDispatch = (behavior: "succeed" | "fail"): Layer.Layer => Layer.succeed(AnalyticsDispatchService, { - dispatchCaptured: () => Effect.void, dispatchTrusted: fakeService(() => { if (behavior === "fail") return Effect.fail(new DispatchBoomError({ message: "dispatch boom" })); return Effect.void; diff --git a/packages/core/test/services/purchaseProcessing/PurchaseProcessingService.integration.test.ts b/packages/core/test/services/purchaseProcessing/PurchaseProcessingService.integration.test.ts index ad7ba1dd5..1cece1a6e 100644 --- a/packages/core/test/services/purchaseProcessing/PurchaseProcessingService.integration.test.ts +++ b/packages/core/test/services/purchaseProcessing/PurchaseProcessingService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PurchaseProcessingService}, run against the real * backend stack provisioned once by `test/_testing/globalSetup.ts` (live - * PlanetScale DB + ClickHouse + WorkOS; only the project schema cache is an + * PostgreSQL; only the project schema cache is an * in-memory stub). * * Unlike the catalog services, `PurchaseProcessingService` is unauthenticated diff --git a/packages/core/test/services/purchases/PurchaseService.integration.test.ts b/packages/core/test/services/purchases/PurchaseService.integration.test.ts index 2f282f64b..0c7556df6 100644 --- a/packages/core/test/services/purchases/PurchaseService.integration.test.ts +++ b/packages/core/test/services/purchases/PurchaseService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link PurchaseService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is an in-memory stub). + * and PostgreSQL; only the project schema cache is an in-memory stub). * * `PurchaseService` exposes a single read-only method, `getPersonPurchases`, * which is gated behind a `project:all` permission check on the *person's* diff --git a/packages/core/test/services/schema/SchemaService.integration.test.ts b/packages/core/test/services/schema/SchemaService.integration.test.ts index 21b63d3d1..01dab5ea2 100644 --- a/packages/core/test/services/schema/SchemaService.integration.test.ts +++ b/packages/core/test/services/schema/SchemaService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link SchemaService}, run against the real backend * stack provisioned once by `test/_testing/globalSetup.ts` (live PlanetScale DB - * + ClickHouse + WorkOS; only the project schema cache is the in-memory stub the + * and PostgreSQL; only the project schema cache is the in-memory stub the * harness builds fresh per test). * * `SchemaService` is a read-only assembler: it folds six concurrent diff --git a/packages/core/test/services/sdk/SdkService.integration.test.ts b/packages/core/test/services/sdk/SdkService.integration.test.ts index a962a6194..aea2b6569 100644 --- a/packages/core/test/services/sdk/SdkService.integration.test.ts +++ b/packages/core/test/services/sdk/SdkService.integration.test.ts @@ -1,7 +1,7 @@ /** * Integration tests for {@link SdkService}, the orchestration hub for every * public SDK route. They run against the real backend stack provisioned once by - * `test/_testing/globalSetup.ts` (live PlanetScale DB + ClickHouse + WorkOS; + * `test/_testing/globalSetup.ts` (live PostgreSQL; * only the project schema cache is an in-memory stub). * * `SdkService` owns no domain logic — it composes `PersonIdentityService`, @@ -124,7 +124,7 @@ const sdkLayer = (appStore: Layer.Layer) => PurchaseService.layer, PersonIdentityService.layer.pipe(Layer.provide(IdentityProjectionPublisher.noop)), // SdkService itself now resolves the publisher (for the synchronous - // person-attribute ClickHouse projection); tests use the no-op. + // person-attribute analytics projection); tests use the no-op. IdentityProjectionPublisher.noop, appStore, googlePlayStub, diff --git a/packages/core/test/services/voidql/compiler.test.ts b/packages/core/test/services/voidql/compiler.test.ts deleted file mode 100644 index 4f2552a7d..000000000 --- a/packages/core/test/services/voidql/compiler.test.ts +++ /dev/null @@ -1,363 +0,0 @@ -/** - * Pure unit tests for the VoidQL compiler — the load-bearing, infra-free core of - * the analytics access layer (docs/analytics-access-layer.html §19). Covers the - * golden text→CH-SQL mapping, the **non-negotiable isolation invariant** (every - * base-ref carries the bound tenant predicate, on the right scope), forbidden- - * keyword rejection, PII gating, out-of-band parameter binding, and the LIMIT - * clamp. No ClickHouse, no Db, no Auth. - */ -import { Predicate, Result } from "effect"; - -import { describe, expect, it } from "../../../src/testing/effect-vitest.ts"; - -import type { Capability } from "../../../src/services/voidql/catalog/types.ts"; -import { compileToIr } from "../../../src/services/voidql/compile.ts"; -import { lit, renderDebugSql } from "../../../src/services/voidql/ir.ts"; -import { makeAuthorizedScope } from "../../../src/services/voidql/scope.ts"; -import { verify } from "../../../src/services/voidql/verify.ts"; - -const SCOPE = makeAuthorizedScope({ - organizationId: "org_a", - availableProjectIds: ["proj_1", "proj_2"], -}); -const PII = new Set(["pii"]); -const NO_PII = new Set([]); - -/** Compile + verify, returning the rendered `(sql, binds)`, columns, and scopes. */ -const compile = (text: string, caps: ReadonlySet = NO_PII, scope = SCOPE) => { - const ir = compileToIr(text, scope, caps); - verify(ir.pieces, ir.injected, scope); - const rendered = renderDebugSql(ir.pieces); - return { sql: rendered.sql, binds: rendered.binds, columns: ir.shape, injected: ir.injected }; -}; - -/** Reads the `_tag` of a thrown value without an `as` assertion. */ -const tagOf = (error: unknown): unknown => { - if (Predicate.hasProperty(error, "_tag")) return error._tag; - return undefined; -}; - -/** Asserts `fn` throws a tagged error carrying `tag`. */ -const expectTag = (fn: () => unknown, tag: string): void => { - const result = Result.try(fn); - expect(Result.isFailure(result)).toBe(true); - if (Result.isFailure(result)) { - expect(tagOf(result.failure)).toBe(tag); - } -}; - -const countOccurrences = (haystack: string, needle: string): number => - haystack.split(needle).length - 1; - -describe("VoidQL compiler — golden mapping", () => { - it("lowers a simple aggregation with the injected tenant scope", () => { - const { sql, binds, columns } = compile( - "SELECT event_name, count() AS n FROM events WHERE event_ts >= '2026-01-01' GROUP BY event_name", - ); - // The raw table is wrapped in a scoped subquery; org + projects are bound. - expect(sql).toContain( - "FROM events_v2 WHERE organization_id = {p1: String} AND project_id IN {p2: Array(String)}", - ); - expect(sql).toContain("LIMIT 1 BY event_id"); - // The identity-join (pending_overrides) is scoped inline too (p3/p4), since the - // analytics_query user has no row policy. - expect(sql).toContain( - "FROM person_identity_pending_overrides_v2 WHERE version > 0 AND organization_id = {p3: String} AND project_id IN {p4: Array(String)}", - ); - // The user's date literal binds out-of-band as DateTime (partition pruning). - expect(sql).toContain("{p5: DateTime}"); - expect(binds).toEqual([ - "org_a", - ["proj_1", "proj_2"], - "org_a", - ["proj_1", "proj_2"], - "2026-01-01 00:00:00", - ]); - expect(sql.trimEnd().endsWith("LIMIT 100000")).toBe(true); - expect(columns).toEqual([ - { name: "event_name", type: "String" }, - { name: "n", type: "UInt64" }, - ]); - }); - - it("binds JSON property keys out-of-band, never spliced", () => { - const { sql, binds } = compile("SELECT count() AS n FROM events WHERE properties.plan = 'pro'"); - // Both the key AND the value are bound params (§9, §18 #1). - expect(sql).toMatch(/JSONExtractString\(events_0\.event_properties, \{p\d+: String\}\)/); - expect(binds).toContain("plan"); - expect(binds).toContain("pro"); - }); - - it("compiles revenue with amount_usd in dollars", () => { - const { sql, columns } = compile("SELECT sum(amount_usd) AS total FROM revenue"); - expect(sql).toContain("event_name IN ('$purchase.completed'"); - expect(sql).toContain("/ 100 AS amount_usd"); - expect(columns).toEqual([{ name: "total", type: "Float64" }]); - }); - - it("clamps an over-large LIMIT to the server cap", () => { - const { sql } = compile("SELECT count() AS n FROM events LIMIT 999999999"); - expect(sql).toContain("LIMIT 100000"); - expect(sql).not.toContain("999999999"); - }); - - it("applies one global result cap around UNION ALL arms", () => { - const { sql } = compile( - "SELECT event_id AS id FROM events UNION ALL SELECT person_id AS id FROM persons", - ); - expect(sql.trimEnd().endsWith("AS voidql_union LIMIT 100000")).toBe(true); - }); - - it("expands SELECT * to in-star catalog columns only (never physical *)", () => { - const { sql, columns } = compile("SELECT * FROM persons", PII); - expect(sql).not.toMatch(/SELECT \*/); - // PII columns are excluded from * even with the capability. - expect(columns.map((c) => c.name)).not.toContain("email"); - expect(columns.map((c) => c.name)).toContain("person_id"); - }); -}); - -describe("VoidQL compiler — tenant isolation invariant", () => { - it("injects exactly one tenant predicate per base-table reference (JOIN)", () => { - const { sql, injected } = compile( - "SELECT e.event_name, count() AS n FROM events AS e " + - "JOIN ( SELECT distinct_id FROM persons WHERE distinct_id != '' ) AS pro " + - "ON pro.distinct_id = e.distinct_id GROUP BY e.event_name", - ); - expect(injected).toHaveLength(2); - expect(countOccurrences(sql, "events_v2")).toBe(1); - expect(countOccurrences(sql, "person_identity_pending_overrides_v2")).toBe(1); - expect(countOccurrences(sql, "persons_v1")).toBe(1); - // events lowering scopes 2 physical reads (events_v2 + pending_overrides), - // persons lowering scopes 1 → 2 + 1 = 3 tenant predicates of each kind. - expect(countOccurrences(sql, "organization_id = {p")).toBe(3); - expect(countOccurrences(sql, "project_id IN {p")).toBe(3); - }); - - it("scopes every base-ref to the authorized org/projects, never the request", () => { - const { injected } = compile( - "SELECT count() AS n FROM events AS a JOIN events AS b ON a.event_id = b.event_id", - ); - expect(injected).toHaveLength(2); - for (const scoped of injected) { - expect(scoped.orgValue).toBe("org_a"); - expect(scoped.projectValues).toEqual(["proj_1", "proj_2"]); - } - }); - - it("the verifier rejects an unscoped base-ref (guard-stubbed leak)", () => { - // Simulate a printer bug that emitted a raw base table with no injected scope. - expectTag( - () => verify([lit("SELECT 1 FROM events_v2 LIMIT 1")], [], SCOPE), - "VoidQlIsolationError", - ); - }); - - it("the verifier rejects a scope bound to a different organization", () => { - const ir = compileToIr("SELECT count() AS n FROM events", SCOPE, NO_PII); - const otherOrg = makeAuthorizedScope({ - organizationId: "org_b", - availableProjectIds: ["proj_1", "proj_2"], - }); - expectTag(() => verify(ir.pieces, ir.injected, otherOrg), "VoidQlIsolationError"); - }); - - it("reserves physical-table-shaped aliases so the verifier never false-positives", () => { - // A user alias like `foo_v1` would otherwise trip the verifier's physical-table - // token scan; such names are rejected up front (unsupported, not isolation). - expectTag(() => compile("SELECT count() AS n FROM events AS evt_v2"), "VoidQlUnsupportedError"); - expectTag(() => compile("SELECT event_id AS id_v1 FROM events"), "VoidQlUnsupportedError"); - expectTag( - () => compile("WITH c_v9 AS ( SELECT event_id FROM events ) SELECT count() AS n FROM c_v9"), - "VoidQlUnsupportedError", - ); - }); - - it("CTEs inject scope at every depth", () => { - const { injected, sql } = compile( - "WITH recent AS ( SELECT event_id, distinct_id FROM events ) " + - "SELECT count() AS n FROM recent", - ); - expect(injected).toHaveLength(1); - expect(countOccurrences(sql, "events_v2")).toBe(1); - }); -}); - -describe("VoidQL compiler — PII gating", () => { - it("rejects a PII column without the capability (never null-substitutes)", () => { - expectTag(() => compile("SELECT email FROM persons"), "VoidQlPiiError"); - expectTag( - () => compile("SELECT count() AS n FROM persons WHERE email = 'x@y.z'"), - "VoidQlPiiError", - ); - expectTag( - () => compile("SELECT count() AS n FROM persons WHERE traits.plan = 'pro'"), - "VoidQlPiiError", - ); - }); - - it("allows PII columns with the capability", () => { - const { sql } = compile("SELECT email, name FROM persons", PII); - expect(sql).toContain("persons_v1"); - }); -}); - -describe("VoidQL compiler — forbidden + unsupported constructs", () => { - it("rejects a trailing SETTINGS clause as ungrammatical", () => { - expectTag( - () => compile("SELECT count() AS n FROM events SETTINGS max_threads = 1"), - "VoidQlUnsupportedError", - ); - }); - - it("rejects ambiguous set syntax and DDL", () => { - expectTag( - () => compile("SELECT count() AS n FROM events UNION SELECT count() FROM events"), - "VoidQlUnsupportedError", - ); - expectTag(() => compile("DROP TABLE events"), "VoidQlUnsupportedError"); - }); - - it("rejects multi-column scalar and IN subqueries", () => { - expectTag( - () => compile("SELECT (SELECT event_id, person_id FROM events) AS invalid"), - "VoidQlUnsupportedError", - ); - expectTag( - () => - compile( - "SELECT count() AS n FROM events WHERE event_id IN (SELECT event_id, person_id FROM events)", - ), - "VoidQlUnsupportedError", - ); - }); - - it("rejects unknown tables, columns, and functions", () => { - expectTag(() => compile("SELECT 1 AS x FROM nope"), "VoidQlSchemaError"); - expectTag(() => compile("SELECT nope FROM events"), "VoidQlUnknownFieldError"); - expectTag(() => compile("SELECT badfn(event_id) AS x FROM events"), "VoidQlUnsupportedError"); - }); - - it("rejects over-deep nesting before recursion blows the stack", () => { - const deep = `SELECT ${"(".repeat(200)}1${")".repeat(200)} AS x FROM events`; - expectTag(() => compile(deep), "VoidQlComplexityError"); - }); -}); - -describe("VoidQL compiler — output column naming (shape ↔ SQL parity)", () => { - it("emits AS for an unaliased computed column so shape matches the CH output name", () => { - const { sql, columns } = compile("SELECT count() FROM events"); - // Without the explicit AS, ClickHouse would name the column `count()` while - // the shape reported `expr_0` — the caller would read the wrong key. - expect(sql).toContain("count() AS expr_0"); - expect(columns).toEqual([{ name: "expr_0", type: "UInt64" }]); - }); - - it("a derived-relation reference to a synthesized column resolves", () => { - // Previously the inner `count()` had no AS, so `s.expr_0` did not exist in the - // subquery and ClickHouse rejected the statement with 'Unknown identifier'. - const { sql, columns } = compile("SELECT expr_0 FROM ( SELECT count() FROM events ) AS s"); - expect(sql).toContain("count() AS expr_0"); - expect(columns).toEqual([{ name: "expr_0", type: "UInt64" }]); - }); -}); - -describe("VoidQL compiler — robustness (typed errors, never raw defects)", () => { - it("rejects an unparseable date literal with a typed error, not a RangeError", () => { - // `new Date('tomorrow').toISOString()` throws a raw RangeError that would escape - // as an opaque defect/500 and break the validate-repair loop (§18 #9). - expectTag( - () => compile("SELECT count() AS n FROM events WHERE event_ts >= 'tomorrow'"), - "VoidQlSyntaxError", - ); - expectTag( - () => compile("SELECT count() AS n FROM events WHERE event_ts = 'not-a-date'"), - "VoidQlSyntaxError", - ); - // A valid ISO date still compiles and binds as a DateTime parameter. - const { binds } = compile("SELECT count() AS n FROM events WHERE event_ts >= '2026-01-01'"); - expect(binds).toContain("2026-01-01 00:00:00"); - }); - - it("bounds a unary-minus chain by depth (typed error, not a stack overflow)", () => { - // The `neg` prefix recurses parsePrefix directly; without a depth guard a long - // `- - … - x` chain blows the JS stack with a raw RangeError before the node cap. - const deep = `SELECT ${"- ".repeat(120)}1 AS x FROM events`; - expectTag(() => compile(deep), "VoidQlComplexityError"); - }); -}); - -describe("VoidQL compiler — verifier backstops (direct IR)", () => { - it("triangulates revenue and persons-only lowerings, not just events", () => { - // Exercise verify() end-to-end for the relations the golden tests skip. - expect(() => compile("SELECT sum(amount_usd) AS total FROM revenue")).not.toThrow(); - expect(() => compile("SELECT count() AS n FROM persons")).not.toThrow(); - const persons = compile("SELECT count() AS n FROM persons"); - expect(countOccurrences(persons.sql, "persons_v1")).toBe(1); - expect(countOccurrences(persons.sql, "organization_id = {p")).toBe(1); - const revenue = compile("SELECT sum(amount_usd) AS total FROM revenue"); - // revenue lowers through the events machinery → 2 scoped physical reads. - expect(countOccurrences(revenue.sql, "organization_id = {p")).toBe(2); - }); - - it("rejects a scope bound to the wrong project set (I1)", () => { - const ir = compileToIr("SELECT count() AS n FROM events", SCOPE, NO_PII); - const wrongProjects = makeAuthorizedScope({ - organizationId: "org_a", - availableProjectIds: ["proj_1"], // missing proj_2 - }); - expectTag(() => verify(ir.pieces, ir.injected, wrongProjects), "VoidQlIsolationError"); - }); - - it("rejects forbidden table-functions / introspection in the emitted SQL (I2)", () => { - for (const token of [ - "remote('h', t)", - "url('http://x')", - "s3('x')", - "getSetting('y')", - "dictGet('d','a',1)", - ]) { - expectTag(() => verify([lit(`SELECT ${token}`)], [], SCOPE), "VoidQlIsolationError"); - } - expectTag( - () => verify([lit("SELECT 1 FROM system.tables")], [], SCOPE), - "VoidQlIsolationError", - ); - }); - - it("rejects an emitted SETTINGS / FORMAT clause (I3 — the override-by-absence net)", () => { - expectTag( - () => verify([lit("SELECT 1 SETTINGS max_threads = 1")], [], SCOPE), - "VoidQlIsolationError", - ); - expectTag(() => verify([lit("SELECT 1 FORMAT JSON")], [], SCOPE), "VoidQlIsolationError"); - }); - - it("rejects a reserved internal alias colliding with the injected machinery", () => { - // `events` / `pending_overrides` are the injected inner aliases; a user alias of - // the same name must be rejected up front (covers the RESERVED branch the - // `_v\\d+` test does not). - expectTag(() => compile("SELECT count() AS n FROM events AS events"), "VoidQlUnsupportedError"); - expectTag( - () => compile("SELECT count() AS n FROM events AS pending_overrides"), - "VoidQlUnsupportedError", - ); - }); -}); - -describe("VoidQL compiler — parameter binding (no escaping class)", () => { - it("binds adversarial string literals as parameters, not raw SQL", () => { - const { sql, binds } = compile( - "SELECT count() AS n FROM events WHERE event_name = ' OR 1=1 --'", - ); - expect(sql).not.toContain("OR 1=1"); - expect(binds).toContain(" OR 1=1 --"); - }); - - it("binds driver-substitution and backslash payloads as values", () => { - const a = compile("SELECT count() AS n FROM events WHERE event_name = '%(x)s'"); - expect(a.binds).toContain("%(x)s"); - const b = compile("SELECT count() AS n FROM events WHERE event_name = 'a\\b'"); - expect(b.binds).toContain("a\\b"); - }); -}); diff --git a/packages/core/test/services/voidql/corpus.ts b/packages/core/test/services/voidql/corpus.ts deleted file mode 100644 index b11c1d517..000000000 --- a/packages/core/test/services/voidql/corpus.ts +++ /dev/null @@ -1,369 +0,0 @@ -/** - * The supported VoidQL query corpus, shared by the compile-only unit suite and - * the live-ClickHouse integration suite so both prove the same surface. - */ -import type { Capability } from "../../../src/services/voidql/catalog/types.ts"; -import { makeAuthorizedScope } from "../../../src/services/voidql/scope.ts"; - -export -const SCOPE = makeAuthorizedScope({ - organizationId: "org_compatibility", - availableProjectIds: ["project_a", "project_b"], -}); -export const CAPABILITIES = new Set(["pii"]); - -export interface SupportedQuery { - readonly name: string; - readonly sql: string; - readonly baseReferences: number; - readonly contains?: readonly string[]; - readonly boundValues?: readonly unknown[]; -} - -export const SUPPORTED_QUERIES: readonly SupportedQuery[] = [ - { - name: "constant SELECT without FROM", - sql: "SELECT 1 AS one, true AS enabled, NULL AS missing", - baseReferences: 0, - }, - { - name: "explicit and implicit projection aliases", - sql: "SELECT event_name AS explicit_name, project_id implicit_project FROM events", - baseReferences: 1, - contains: ["AS explicit_name", "AS implicit_project"], - }, - { - name: "expression alias reused by a later projection", - sql: "SELECT upper(event_name) AS normalized, length(normalized) AS normalized_length FROM events", - baseReferences: 1, - contains: ["upper(", "length(normalized)"], - }, - { - name: "projection alias reused in WHERE", - sql: "SELECT lower(event_name) AS normalized FROM events WHERE normalized LIKE 'purchase%'", - baseReferences: 1, - contains: ["WHERE (normalized LIKE"], - boundValues: ["purchase%"], - }, - { - name: "projection aliases reused in GROUP BY, HAVING, and ORDER BY", - sql: "SELECT toStartOfDay(event_ts) AS day, count() AS total FROM events GROUP BY day HAVING total > 1 ORDER BY total DESC", - baseReferences: 1, - contains: ["GROUP BY day", "HAVING (total >", "ORDER BY total DESC"], - }, - { - name: "explicit table alias", - sql: "SELECT e.event_id FROM events AS e", - baseReferences: 1, - contains: ["AS e", "e.event_id"], - }, - { - name: "implicit table alias", - sql: "SELECT e.event_id FROM events e", - baseReferences: 1, - contains: ["AS e", "e.event_id"], - }, - { - name: "SELECT ALL", - sql: "SELECT ALL event_name FROM events", - baseReferences: 1, - }, - { - name: "SELECT DISTINCT", - sql: "SELECT DISTINCT event_name FROM events", - baseReferences: 1, - contains: ["SELECT DISTINCT"], - }, - { - name: "SELECT DISTINCT ON", - sql: "SELECT DISTINCT ON (person_id) person_id, event_ts FROM events ORDER BY event_ts DESC", - baseReferences: 1, - contains: ["SELECT DISTINCT ON (", "ORDER BY"], - }, - { - name: "arithmetic precedence and unary negation", - sql: "SELECT -(1 + 2 * 3) AS result FROM events LIMIT 1", - baseReferences: 1, - }, - { - name: "boolean comparison precedence", - sql: "SELECT event_id FROM events WHERE event_name = 'a' OR event_name = 'b' AND project_id != 'c'", - baseReferences: 1, - boundValues: ["a", "b", "c"], - }, - { - name: "LIKE and NOT LIKE", - sql: "SELECT event_id FROM events WHERE event_name LIKE 'a%' AND event_name NOT LIKE '%bot%'", - baseReferences: 1, - boundValues: ["a%", "%bot%"], - }, - { - name: "ILIKE and NOT ILIKE", - sql: "SELECT event_id FROM events WHERE event_name ILIKE 'a%' AND event_name NOT ILIKE '%bot%'", - baseReferences: 1, - contains: [" ILIKE ", " NOT ILIKE "], - boundValues: ["a%", "%bot%"], - }, - { - name: "IN and NOT IN literal lists", - sql: "SELECT event_id FROM events WHERE event_name IN ('a', 'b') AND project_id NOT IN ('x', 'y')", - baseReferences: 1, - boundValues: ["a", "b", "x", "y"], - }, - { - name: "BETWEEN and NOT BETWEEN", - sql: "SELECT event_id FROM events WHERE event_ts BETWEEN '2026-01-01' AND '2026-01-31' AND event_name NOT BETWEEN 'a' AND 'z'", - baseReferences: 1, - }, - { - name: "IS NULL and IS NOT NULL", - sql: "SELECT event_id FROM events WHERE person_id IS NULL OR distinct_id IS NOT NULL", - baseReferences: 1, - }, - { - name: "searched and simple CASE", - sql: "SELECT CASE WHEN event_name = 'a' THEN 'first' ELSE 'other' END AS searched, CASE event_name WHEN 'b' THEN 'second' ELSE 'other' END AS simple FROM events", - baseReferences: 1, - boundValues: ["a", "first", "b", "second", "other"], - }, - { - name: "PREWHERE lowered safely ahead of WHERE", - sql: "SELECT event_id FROM events PREWHERE event_ts >= '2026-01-01' WHERE event_name = 'signup'", - baseReferences: 1, - contains: [" WHERE ", " AND "], - boundValues: ["signup"], - }, - { - name: "GROUP BY and HAVING", - sql: "SELECT event_name, count() AS total FROM events GROUP BY event_name HAVING count() > 10", - baseReferences: 1, - contains: ["GROUP BY", "HAVING"], - }, - { - name: "GROUP BY WITH ROLLUP and TOTALS", - sql: "SELECT event_name, count() AS total FROM events GROUP BY event_name WITH ROLLUP WITH TOTALS", - baseReferences: 1, - contains: ["WITH ROLLUP", "WITH TOTALS"], - }, - { - name: "GROUP BY WITH CUBE", - sql: "SELECT event_name, project_id, count() AS total FROM events GROUP BY event_name, project_id WITH CUBE", - baseReferences: 1, - contains: ["WITH CUBE"], - }, - { - name: "multi-column ORDER BY", - sql: "SELECT event_name, event_ts FROM events ORDER BY event_name ASC, event_ts DESC", - baseReferences: 1, - contains: ["ORDER BY", " ASC", " DESC"], - }, - { - name: "ORDER BY NULLS FIRST and NULLS LAST", - sql: "SELECT event_name, event_ts FROM events ORDER BY event_name ASC NULLS FIRST, event_ts DESC NULLS LAST", - baseReferences: 1, - contains: ["NULLS FIRST", "NULLS LAST"], - }, - { - name: "row-number window with partition and ordering", - sql: "SELECT event_id, rowNumber() OVER (PARTITION BY person_id ORDER BY event_ts DESC) AS row_num FROM events", - baseReferences: 1, - contains: ["row_number() OVER (PARTITION BY", "ORDER BY"], - }, - { - name: "aggregate window with an unbounded frame", - sql: "SELECT event_id, sum(1) OVER (PARTITION BY person_id ORDER BY event_ts ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running_total FROM events", - baseReferences: 1, - contains: ["ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW"], - }, - { - name: "QUALIFY over a window alias", - sql: "SELECT event_id, rowNumber() OVER (PARTITION BY person_id ORDER BY event_ts DESC) AS row_num FROM events QUALIFY row_num = 1", - baseReferences: 1, - contains: [" QUALIFY (row_num ="], - }, - { - name: "LIMIT count OFFSET offset", - sql: "SELECT event_id FROM events LIMIT 10 OFFSET 5", - baseReferences: 1, - contains: ["LIMIT 10 OFFSET 5"], - }, - { - name: "LIMIT offset, count", - sql: "SELECT event_id FROM events LIMIT 5, 10", - baseReferences: 1, - contains: ["LIMIT 10 OFFSET 5"], - }, - { - name: "LIMIT WITH TIES", - sql: "SELECT event_name FROM events ORDER BY event_name LIMIT 10 WITH TIES", - baseReferences: 1, - contains: ["LIMIT 10 WITH TIES"], - }, - { - name: "LIMIT BY plus result LIMIT", - sql: "SELECT event_name, event_ts FROM events ORDER BY event_ts DESC LIMIT 2 BY event_name LIMIT 10", - baseReferences: 1, - contains: ["LIMIT 2 BY", "LIMIT 10"], - }, - { - name: "INNER JOIN with ON", - sql: "SELECT e.event_id FROM events AS e INNER JOIN persons AS p ON p.person_id = e.person_id", - baseReferences: 2, - contains: ["INNER JOIN", " ON "], - }, - { - name: "LEFT OUTER JOIN", - sql: "SELECT e.event_id FROM events AS e LEFT OUTER JOIN persons AS p ON p.person_id = e.person_id", - baseReferences: 2, - contains: ["LEFT JOIN"], - }, - { - name: "RIGHT OUTER JOIN", - sql: "SELECT e.event_id FROM events AS e RIGHT OUTER JOIN persons AS p ON p.person_id = e.person_id", - baseReferences: 2, - contains: ["RIGHT JOIN"], - }, - { - name: "FULL OUTER JOIN", - sql: "SELECT e.event_id FROM events AS e FULL OUTER JOIN persons AS p ON p.person_id = e.person_id", - baseReferences: 2, - contains: ["FULL JOIN"], - }, - { - name: "CROSS JOIN", - sql: "SELECT count() AS combinations FROM events AS e CROSS JOIN persons AS p", - baseReferences: 2, - contains: ["CROSS JOIN"], - }, - { - name: "JOIN USING", - sql: "SELECT e.event_name FROM events AS e JOIN persons AS p USING (project_id)", - baseReferences: 2, - contains: ["USING (project_id)"], - }, - { - name: "derived-table subquery", - sql: "SELECT recent.event_id FROM (SELECT event_id FROM events WHERE event_ts >= '2026-01-01') AS recent", - baseReferences: 1, - }, - { - name: "scalar subquery", - sql: "SELECT (SELECT count() FROM persons) AS person_count", - baseReferences: 1, - contains: ["( SELECT count()"], - }, - { - name: "IN subquery", - sql: "SELECT event_id FROM events WHERE event_id IN (SELECT event_id FROM events WHERE event_name = 'signup')", - baseReferences: 2, - boundValues: ["signup"], - }, - { - name: "EXISTS subquery", - sql: "SELECT e.event_id FROM events AS e WHERE EXISTS (SELECT p.person_id FROM persons AS p WHERE p.project_id = 'project_a')", - baseReferences: 2, - contains: ["EXISTS ( SELECT"], - boundValues: ["project_a"], - }, - { - name: "NOT EXISTS subquery", - sql: "SELECT e.event_id FROM events AS e WHERE NOT EXISTS (SELECT p.person_id FROM persons AS p WHERE p.project_id = 'project_a')", - baseReferences: 2, - contains: ["NOT EXISTS"], - boundValues: ["project_a"], - }, - { - name: "single CTE", - sql: "WITH recent AS (SELECT event_id FROM events WHERE event_ts >= '2026-01-01') SELECT count() AS total FROM recent", - baseReferences: 1, - contains: ["WITH recent AS"], - }, - { - name: "dependent CTEs", - sql: "WITH base AS (SELECT event_id FROM events), copied AS (SELECT event_id FROM base) SELECT count() AS total FROM copied", - baseReferences: 1, - contains: ["WITH base AS", ", copied AS"], - }, - { - name: "UNION ALL", - sql: "SELECT event_id AS id FROM events UNION ALL SELECT person_id AS id FROM persons", - baseReferences: 2, - contains: [" UNION ALL ", ") AS voidql_union LIMIT 100000"], - }, - { - name: "three UNION ALL arms", - sql: "SELECT event_id AS id FROM events UNION ALL SELECT person_id AS id FROM persons UNION ALL SELECT event_id AS id FROM revenue", - baseReferences: 3, - contains: [" UNION ALL "], - }, - { - name: "UNION DISTINCT", - sql: "SELECT event_id AS id FROM events UNION DISTINCT SELECT person_id AS id FROM persons", - baseReferences: 2, - contains: [" UNION DISTINCT "], - }, - { - name: "INTERSECT", - sql: "SELECT project_id FROM events INTERSECT SELECT project_id FROM persons", - baseReferences: 2, - contains: [" INTERSECT "], - }, - { - name: "EXCEPT", - sql: "SELECT project_id FROM events EXCEPT SELECT project_id FROM persons", - baseReferences: 2, - contains: [" EXCEPT "], - }, - { - name: "UNION ALL inside a CTE", - sql: "WITH ids AS (SELECT event_id AS id FROM events UNION ALL SELECT person_id AS id FROM persons) SELECT count() AS total FROM ids", - baseReferences: 2, - }, - { - name: "aggregate function family", - sql: "SELECT count(), countIf(event_name = 'a'), countDistinct(person_id), sum(1), sumIf(1, event_name = 'a'), avg(1), min(event_ts), max(event_ts), any(event_name), argMin(event_name, event_ts), argMax(event_name, event_ts) FROM events", - baseReferences: 1, - }, - { - name: "conditional function family", - sql: "SELECT if(event_name = 'a', 'yes', 'no'), multiIf(event_name = 'a', 'a', event_name = 'b', 'b', 'other'), coalesce(person_id, distinct_id), nullIf(person_id, ''), ifNull(person_id, ''), greatest(1, 2), least(1, 2) FROM events", - baseReferences: 1, - }, - { - name: "string function family", - sql: "SELECT lower(event_name), upper(event_name), length(event_name), trim(event_name), concat(event_name, project_id), substring(event_name, 1, 3), startsWith(event_name, 'a'), endsWith(event_name, 'z'), position(event_name, 'x'), replaceAll(event_name, 'a', 'b'), match(event_name, '^a'), replaceRegexpAll(event_name, 'a', 'b'), cityHash64(event_id) FROM events", - baseReferences: 1, - }, - { - name: "date function family", - sql: "SELECT toStartOfMinute(event_ts), toStartOfHour(event_ts), toStartOfDay(event_ts), toStartOfWeek(event_ts), toStartOfMonth(event_ts), toStartOfQuarter(event_ts), toStartOfYear(event_ts), toDate(event_ts), dateDiff('day', event_ts, event_ts), toYear(event_ts), toMonth(event_ts), toDayOfWeek(event_ts) FROM events", - baseReferences: 1, - }, - { - name: "math and safe cast function family", - sql: "SELECT round(amount_usd, 2), floor(amount_usd), ceil(amount_usd), abs(amount_usd), sqrt(amount_usd), pow(amount_usd, 2), exp(amount_usd), log(amount_usd), toFloat64OrNull(properties.amount), toInt64OrNull(properties.quantity), toDateOrNull(properties.date) FROM revenue", - baseReferences: 1, - }, - { - name: "JSON property namespaces", - sql: "SELECT properties.plan, context.locale FROM events WHERE properties.plan = 'pro'", - baseReferences: 1, - boundValues: ["plan", "locale", "pro"], - }, - { - name: "PII columns and traits with capability", - sql: "SELECT email, name, traits.plan FROM persons WHERE traits.plan = 'pro'", - baseReferences: 1, - boundValues: ["plan", "pro"], - }, - { - name: "qualified catalog star", - sql: "SELECT e.* FROM events AS e", - baseReferences: 1, - }, - { - name: "comments and trailing semicolon", - sql: "/* leading */ SELECT event_id -- projection\nFROM events;", - baseReferences: 1, - }, -]; diff --git a/packages/core/test/services/voidql/query-compatibility.integration.test.ts b/packages/core/test/services/voidql/query-compatibility.integration.test.ts deleted file mode 100644 index 07bc190c8..000000000 --- a/packages/core/test/services/voidql/query-compatibility.integration.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Executes the supported VoidQL corpus against a live ClickHouse, proving the - * compiled SQL is accepted by the real parameter substrate rather than only by - * the compiler's own model of it. - * - * Previously an environment-gated block inside the unit suite, where it never - * ran: the flag was set by no tier. - * - * The connection comes from the harness — {@link ClickhouseWebClient} bound to - * the injected test connections — rather than from the environment, so the case - * runs unchanged under every composition: the self-host stack locally, a - * provisioned deployment downstream. Only acceptance is asserted, so the - * harness's read-write binding proves what a least-privilege one would: every - * compiled statement parses, resolves, and binds its parameters. - */ -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { Data, Effect } from "effect"; -import { describe } from "vitest"; - -import { CoreIntegrationTestHarness } from "@testing/CoreIntegrationTestHarness"; - -import { compileToIr } from "../../../src/services/voidql/compile.ts"; -import { toStatement } from "../../../src/services/voidql/ir.ts"; -import { CAPABILITIES, SCOPE, SUPPORTED_QUERIES } from "./corpus.ts"; - -const { test } = CoreIntegrationTestHarness.make(); - -/** Local-only failure: never crosses an RPC/queue boundary. */ -class VoidQlCompatibilityError extends Data.TaggedError("VoidQlCompatibilityError")<{ - readonly message: string; - readonly cause: unknown; -}> {} - -describe("VoidQL live ClickHouse compatibility", () => { - test( - "executes every supported query through the real parameter substrate", - Effect.gen(function* () { - const ch = yield* ClickhouseWebClient.ClickhouseWebClient; - - for (const testCase of SUPPORTED_QUERIES) { - const compiled = compileToIr(testCase.sql, SCOPE, CAPABILITIES); - yield* toStatement(ch, compiled.pieces).pipe( - Effect.mapError( - (cause) => - new VoidQlCompatibilityError({ - cause, - message: `ClickHouse rejected compatibility case '${testCase.name}'.`, - }), - ), - ); - } - }), - ); -}); diff --git a/packages/core/test/services/voidql/query-compatibility.test.ts b/packages/core/test/services/voidql/query-compatibility.test.ts deleted file mode 100644 index 617f86e29..000000000 --- a/packages/core/test/services/voidql/query-compatibility.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -/** - * The supported VoidQL query corpus. Every accepted query is compiled through the - * tenant verifier, so adding syntax here also proves that all newly reachable base - * relations remain scoped. Keep this file exhaustive for the intentionally exposed - * ClickHouse SELECT surface; administrative, mutating, external-source, and - * settings/output clauses belong in the permanent rejection corpus below. - */ -import { constant } from "@voidhash/lib/lang"; - -import { describe, expect, it } from "../../../src/testing/effect-vitest.ts"; - -import { compileToIr } from "../../../src/services/voidql/compile.ts"; -import { registeredFunctionNames } from "../../../src/services/voidql/functions.ts"; -import { renderDebugSql } from "../../../src/services/voidql/ir.ts"; -import { verify } from "../../../src/services/voidql/verify.ts"; -import { - CAPABILITIES, - SCOPE, - SUPPORTED_QUERIES, -} from "./corpus.ts"; - -describe("VoidQL supported ClickHouse SELECT compatibility corpus", () => { - for (const testCase of SUPPORTED_QUERIES) { - it(testCase.name, () => { - const compiled = compileToIr(testCase.sql, SCOPE, CAPABILITIES); - verify(compiled.pieces, compiled.injected, SCOPE); - const rendered = renderDebugSql(compiled.pieces); - - expect(compiled.injected).toHaveLength(testCase.baseReferences); - expect(rendered.sql).not.toMatch(/\bSETTINGS\b|\bFORMAT\b/i); - for (const fragment of testCase.contains ?? []) expect(rendered.sql).toContain(fragment); - for (const value of testCase.boundValues ?? []) expect(rendered.binds).toContainEqual(value); - }); - } -}); - -describe("VoidQL function allowlist snapshot", () => { - it("lists every callable function intentionally", () => { - expect([...registeredFunctionNames()].sort()).toEqual( - [ - "abs", - "any", - "argmax", - "argmin", - "avg", - "ceil", - "cityhash64", - "coalesce", - "concat", - "count", - "countdistinct", - "countif", - "datediff", - "denserank", - "endswith", - "exp", - "floor", - "greatest", - "if", - "ifnull", - "least", - "length", - "log", - "lower", - "match", - "max", - "min", - "multiif", - "nullif", - "position", - "pow", - "replaceall", - "replaceregexpall", - "rank", - "round", - "rownumber", - "sqrt", - "startswith", - "substring", - "sum", - "sumif", - "todateornull", - "todate", - "tofloat64ornull", - "toint64ornull", - "tomonth", - "tostartofday", - "tostartofhour", - "tostartofminute", - "tostartofmonth", - "tostartofquarter", - "tostartofweek", - "tostartofyear", - "todayofweek", - "toyear", - "trim", - "upper", - ].sort(), - ); - }); -}); - -const PERMANENTLY_REJECTED = constant([ - "SELECT event_id FROM events SETTINGS max_threads = 1", - "SELECT event_id FROM events FORMAT JSON", - "SELECT event_id FROM events INTO OUTFILE 'result.csv'", - "SELECT * FROM url('https://example.com/data.csv')", - "SELECT * FROM system.tables", - "SELECT currentUser() AS user", - "SELECT event_id FROM events FINAL", - "SELECT event_id FROM events SAMPLE 0.1", - "INSERT INTO events VALUES (1)", - "UPDATE events SET event_name = 'x'", - "DELETE FROM events WHERE true", - "DROP TABLE events", - "SELECT event_id FROM events UNION SELECT event_id FROM events", - "SELECT e.event_id FROM events AS e WHERE EXISTS (SELECT p.person_id FROM persons AS p WHERE p.person_id = e.person_id)", - "SELECT sum(1) OVER (ORDER BY event_ts ROWS BETWEEN 2 PRECEDING AND CURRENT ROW) FROM events", - "SELECT rowNumber() FROM events", - "SELECT e.*, p.* FROM events AS e JOIN persons AS p ON e.person_id = p.person_id", -]); - -describe("VoidQL permanent security boundary", () => { - for (const sql of PERMANENTLY_REJECTED) { - it(`rejects ${sql}`, () => { - expect(() => compileToIr(sql, SCOPE, CAPABILITIES)).toThrow(); - }); - } -}); diff --git a/packages/core/test/services/voidql/substrate.test.ts b/packages/core/test/services/voidql/substrate.test.ts deleted file mode 100644 index 1352bfa94..000000000 --- a/packages/core/test/services/voidql/substrate.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Cross-checks that the pure {@link SqlPiece} IR replays faithfully through the - * real `ch` substrate: `toStatement(ch, pieces).compile()` must reproduce exactly - * the SQL text + ordered binds that {@link renderDebugSql} predicts. This ties the - * verifier's view of the query (the IR) to the bytes that actually reach ClickHouse - * — every user value a bound `{pN:Type}` placeholder, never spliced. - * - * Uses `ClickhouseWebClient.makeUnchecked`, which builds no connection until the - * first query; `.compile()` is pure, so no ClickHouse is contacted. - */ -import { ClickhouseWebClient } from "@voidhash/clickhouse-db/clickhouse-client-web"; -import { Effect } from "effect"; -import * as Reactivity from "effect/unstable/reactivity/Reactivity"; - -import { describe, expect, it } from "../../../src/testing/effect-vitest.ts"; - -import type { Capability } from "../../../src/services/voidql/catalog/types.ts"; -import { compileToIr } from "../../../src/services/voidql/compile.ts"; -import { renderDebugSql, toStatement } from "../../../src/services/voidql/ir.ts"; -import { makeAuthorizedScope } from "../../../src/services/voidql/scope.ts"; - -const SCOPE = makeAuthorizedScope({ - organizationId: "org_a", - availableProjectIds: ["proj_1", "proj_2"], -}); - -const makeClient = ClickhouseWebClient.makeUnchecked(() => ({ - url: "http://localhost:8123", -})).pipe(Effect.provide(Reactivity.layer)); - -describe("VoidQL IR ↔ ch substrate", () => { - it.effect("toStatement().compile() reproduces the IR's SQL text and binds exactly", () => - Effect.gen(function* () { - const ch = yield* makeClient; - const cases = [ - "SELECT event_name, count() AS n FROM events WHERE event_ts >= '2026-01-01' GROUP BY event_name", - "SELECT count() AS n FROM events WHERE properties.plan = 'pro'", - "SELECT e.event_name FROM events AS e JOIN ( SELECT distinct_id FROM persons WHERE distinct_id != '' ) AS p ON p.distinct_id = e.distinct_id", - ]; - for (const text of cases) { - const ir = compileToIr(text, SCOPE, new Set(["pii"])); - const expected = renderDebugSql(ir.pieces); - const [sql, params] = toStatement(ch, ir.pieces).compile(); - expect(sql).toBe(expected.sql); - expect(params).toEqual(expected.binds); - } - }), - ); -}); diff --git a/packages/db/src/alchemy-migrations/20260809120000_add_portable_analytics_events/migration.sql b/packages/db/src/alchemy-migrations/20260809120000_add_portable_analytics_events/migration.sql new file mode 100644 index 000000000..9efe3d408 --- /dev/null +++ b/packages/db/src/alchemy-migrations/20260809120000_add_portable_analytics_events/migration.sql @@ -0,0 +1,37 @@ +CREATE TABLE "analytics_event" ( + "sequence" bigserial PRIMARY KEY NOT NULL, + "schema_version" smallint DEFAULT 1 NOT NULL, + "event_id" varchar(255) NOT NULL, + "capture_id" varchar(255) NOT NULL, + "event_name" varchar(255) NOT NULL, + "event_timestamp" timestamp(3) with time zone NOT NULL, + "processed_at" timestamp(3) with time zone DEFAULT now() NOT NULL, + "organization_id" varchar(255) NOT NULL, + "project_id" varchar(255) NOT NULL, + "distinct_id" varchar(512) NOT NULL, + "previous_distinct_id" varchar(512), + "person_id" varchar(255), + "identity_mode" varchar(32) NOT NULL, + "properties" jsonb NOT NULL, + "context" jsonb NOT NULL, + "session_id" varchar(255), + "token" varchar(255) NOT NULL, + "request_id" varchar(255) NOT NULL, + "request_path" varchar(255), + "source" varchar(32) NOT NULL, + "source_topic" varchar(255) NOT NULL, + CONSTRAINT "analytics_event_project_id_project_id_fk" + FOREIGN KEY ("project_id") REFERENCES "public"."project"("id") + ON DELETE cascade ON UPDATE no action +); + +CREATE UNIQUE INDEX "analytics_event_project_event_uidx" + ON "analytics_event" USING btree ("project_id", "event_id"); +CREATE INDEX "analytics_event_project_time_idx" + ON "analytics_event" USING btree ("project_id", "event_timestamp"); +CREATE INDEX "analytics_event_org_time_idx" + ON "analytics_event" USING btree ("organization_id", "event_timestamp"); +CREATE INDEX "analytics_event_project_name_time_idx" + ON "analytics_event" USING btree ("project_id", "event_name", "event_timestamp"); +CREATE INDEX "analytics_event_export_cursor_idx" + ON "analytics_event" USING btree ("sequence"); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index f690a85e7..62b41ca64 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -3,6 +3,7 @@ import { constant } from "@voidhash/lib/lang"; import { sql } from "drizzle-orm"; import { bigint, + bigserial, boolean, index, integer, @@ -159,8 +160,8 @@ export const projects = pgTable( name: varchar("name", { length: 255 }).notNull(), organizationId: varchar("organization_id", { length: 255 }).notNull(), slug: varchar("slug", { length: 255 }).notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -194,13 +195,60 @@ export const captureProjectPolicies = pgTable( .notNull() .default(48), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [index("capture_project_policy_force_route_idx").on(table.forceRoute)], ); +/** + * Portable analytics event log used by the Community edition. Its semantic + * columns mirror the hosted analytics event record so rows can be exported and + * imported without reconstructing events from application tables. + */ +export const analyticsEvents = pgTable( + "analytics_event", + { + sequence: bigserial("sequence", { mode: "number" }).primaryKey(), + schemaVersion: smallint("schema_version").notNull().default(1), + eventId: varchar("event_id", { length: 255 }).notNull(), + captureId: varchar("capture_id", { length: 255 }).notNull(), + eventName: varchar("event_name", { length: 255 }).notNull(), + eventTimestamp: timestamp("event_timestamp", { withTimezone: true, precision: 3 }).notNull(), + processedAt: timestamp("processed_at", { withTimezone: true, precision: 3 }) + .notNull() + .defaultNow(), + organizationId: varchar("organization_id", { length: 255 }).notNull(), + projectId: varchar("project_id", { length: 255 }) + .notNull() + .references(() => projects.id, { onDelete: "cascade" }), + distinctId: varchar("distinct_id", { length: 512 }).notNull(), + previousDistinctId: varchar("previous_distinct_id", { length: 512 }), + personId: varchar("person_id", { length: 255 }), + identityMode: varchar("identity_mode", { length: 32 }).notNull(), + properties: jsonb("properties").$type>>().notNull(), + context: jsonb("context").$type>>().notNull(), + sessionId: varchar("session_id", { length: 255 }), + token: varchar("token", { length: 255 }).notNull(), + requestId: varchar("request_id", { length: 255 }).notNull(), + requestPath: varchar("request_path", { length: 255 }), + source: varchar("source", { length: 32 }).notNull(), + sourceTopic: varchar("source_topic", { length: 255 }).notNull(), + }, + (table) => [ + uniqueIndex("analytics_event_project_event_uidx").on(table.projectId, table.eventId), + index("analytics_event_project_time_idx").on(table.projectId, table.eventTimestamp), + index("analytics_event_org_time_idx").on(table.organizationId, table.eventTimestamp), + index("analytics_event_project_name_time_idx").on( + table.projectId, + table.eventName, + table.eventTimestamp, + ), + index("analytics_event_export_cursor_idx").on(table.sequence), + ], +); + export const apiKeys = pgTable( "api_key", { @@ -230,8 +278,8 @@ export const apiKeys = pgTable( */ projectId: varchar("project_id", { length: 255 }).notNull(), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [index("api_key_key_idx").on(table.key)], @@ -330,8 +378,8 @@ export const persons = pgTable( deletedAt: timestamp("deleted_at", { withTimezone: true, precision: 3 }), deletionReason: varchar("deletion_reason", { length: 64 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [index("person_merged_into_person_id_idx").on(table.mergedIntoPersonId)], @@ -351,8 +399,8 @@ export const personIdentities = pgTable( kind: smallint("kind").notNull().default(PersonIdentityKind.Anonymous), version: integer("version").notNull().default(0), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -370,8 +418,8 @@ export const personPersonlessIdentities = pgTable( distinctId: varchar("distinct_id", { length: 255 }).notNull(), isMerged: boolean("is_merged").notNull().default(false), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -446,8 +494,8 @@ export const personIdentityMigrationJobs = pgTable( requestedAt: timestamp("requested_at", { withTimezone: true, precision: 3 }).notNull(), completedAt: timestamp("completed_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -488,8 +536,8 @@ export const personUnlockedPerks = pgTable( }), expiresAt: timestamp("expires_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [uniqueIndex("person_id_perk_id_idx").on(table.personId, table.perkId)], @@ -509,8 +557,8 @@ export const personExternalIdentifiers = pgTable( isDefault: boolean("is_default").notNull(), identifier: varchar("identifier", { length: 255 }).notNull(), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -549,8 +597,8 @@ export const personDeletionRequests = pgTable( .defaultNow(), completedAt: timestamp("completed_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -573,8 +621,8 @@ export const paymentProviderConfigurations = pgTable( name: varchar("name", { length: 255 }).notNull().default("Unknown"), configuration: jsonb("configuration").$type(), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), deletedAt: timestamp("deleted_at", { withTimezone: true, precision: 3 }), activeProviderId: varchar("active_provider_id", { length: 255 }).generatedAlwaysAs( @@ -600,8 +648,8 @@ export const perks = pgTable( name: varchar("name", { length: 255 }).notNull(), projectId: varchar("project_id", { length: 255 }).notNull(), slug: varchar("slug", { length: 255 }).notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [uniqueIndex("perk_slug_project_id_idx").on(table.slug, table.projectId)], @@ -616,8 +664,8 @@ export const products = pgTable( projectId: varchar("project_id", { length: 255 }).notNull(), slug: varchar("slug", { length: 255 }).notNull(), type: smallint("type").notNull().default(ProductType.Subscription), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [uniqueIndex("product_slug_project_id_idx").on(table.slug, table.projectId)], @@ -630,8 +678,8 @@ export const productPerks = pgTable( id: varchar("id", { length: 255 }).primaryKey(), perkId: varchar("perk_id", { length: 255 }).notNull(), productId: varchar("product_id", { length: 255 }).notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [uniqueIndex("product_id_perk_id_idx").on(table.productId, table.perkId)], @@ -651,8 +699,8 @@ export const paymentProviderConfigurationProducts = pgTable( providerProductKey: varchar("provider_product_key", { length: 255, }).notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -687,8 +735,8 @@ export const checkoutSessions = pgTable("checkout_session", { }) .notNull() .default("LEGACY"), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }); @@ -732,8 +780,8 @@ export const purchases = pgTable( lastEventOccurredAt: timestamp("last_event_occurred_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -838,8 +886,8 @@ export const subscriptions = pgTable( redeemedOfferAt: timestamp("redeemed_offer_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -860,7 +908,7 @@ export const transactions = pgTable( personId: varchar("person_id", { length: 255 }).notNull(), /** * Legacy single-amount column. Kept temporarily for read-side compatibility - * while ClickHouse views migrate to {@link transactions.grossAmount}. + * while analytics readers migrate to {@link transactions.grossAmount}. * New writes mirror `grossAmount` into this column; remove once readers * have cut over. */ @@ -957,8 +1005,8 @@ export const transactions = pgTable( */ lastEventOccurredAt: timestamp("last_event_occurred_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -1082,8 +1130,8 @@ export const purchaseLedger = pgTable( claimedAt: timestamp("claimed_at", { withTimezone: true, precision: 3 }), publishedAt: timestamp("published_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -1121,8 +1169,8 @@ export const analyticsIngestDlq = pgTable( .notNull() .default(AnalyticsIngestDlqReplayStatus.Pending), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).notNull().defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -1331,8 +1379,8 @@ export const paywalls = pgTable( // render from overwriting a newer one (see PaywallThumbnailService). thumbnailUrl: text("thumbnail_url"), thumbnailSeq: bigint("thumbnail_seq", { mode: "number" }), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [uniqueIndex("paywall_slug_project_id_idx").on(table.slug, table.projectId)], @@ -1520,8 +1568,8 @@ export const paywallLocations = pgTable( description: varchar("description", { length: 1000 }), archivedAt: timestamp("archived_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -1547,8 +1595,8 @@ export const paywallLocationShowings = pgTable( endedAt: timestamp("ended_at", { withTimezone: true, precision: 3 }), createdByUserId: varchar("created_by_user_id", { length: 255 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -1793,8 +1841,8 @@ export const webhookEndpoints = pgTable( lastSuccessAt: timestamp("last_success_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [index("webhook_endpoint_project_status_idx").on(table.projectId, table.status)], @@ -2030,8 +2078,8 @@ export const featureFlags = pgTable( createdByUserId: varchar("created_by_user_id", { length: 255 }), updatedByUserId: varchar("updated_by_user_id", { length: 255 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), version: integer("version").notNull().default(1), }, @@ -2051,8 +2099,8 @@ export const featureFlagTargets = pgTable( identityValue: varchar("identity_value", { length: 255 }).notNull(), archivedAt: timestamp("archived_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -2082,8 +2130,8 @@ export const featureFlagOverrides = pgTable( updatedByUserId: varchar("updated_by_user_id", { length: 255 }), archivedAt: timestamp("archived_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -2107,8 +2155,8 @@ export const featureFlagVariants = pgTable( payload: jsonb("payload").$type(), archivedAt: timestamp("archived_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -2137,8 +2185,8 @@ export const internalFeatureFlagOverrides = pgTable( flagKey: varchar("flag_key", { length: 100 }).notNull(), enabled: boolean("enabled").notNull(), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -2339,8 +2387,8 @@ export const experiments = pgTable( updatedByUserId: varchar("updated_by_user_id", { length: 255 }), archivedAt: timestamp("archived_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), version: integer("version").notNull().default(1), }, @@ -2367,8 +2415,8 @@ export const experimentVariants = pgTable( weightBps: integer("weight_bps").notNull().default(0), archivedAt: timestamp("archived_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [index("experiment_variant_experiment_id_idx").on(table.experimentId)], @@ -2410,8 +2458,8 @@ export const experimentTreatments = pgTable( config: jsonb("config").$type().notNull(), archivedAt: timestamp("archived_at", { withTimezone: true, precision: 3 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), }, (table) => [ @@ -2649,8 +2697,8 @@ export const pushNotificationConfigs = pgTable( name: varchar("name", { length: 255 }).notNull().default("Unknown"), configuration: jsonb("configuration").$type>().notNull(), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow().notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), deletedAt: timestamp("deleted_at", { withTimezone: true, precision: 3 }), activeProviderId: varchar("active_provider_id", { length: 50 }).generatedAlwaysAs( @@ -2688,8 +2736,8 @@ export const pushDeviceTokens = pgTable( invalidationReason: varchar("invalidation_reason", { length: 100 }), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow().notNull(), // Freshness clock for invalidation gating — see NotificationTokenService.invalidate. - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), deletedAt: timestamp("deleted_at", { withTimezone: true, precision: 3 }), }, @@ -2718,8 +2766,8 @@ export const pushPersonDeviceTokens = pgTable( personId: varchar("person_id", { length: 255 }).notNull(), // references persons.id; re-pointed on merge pushDeviceTokenId: varchar("push_device_token_id", { length: 255 }).notNull(), createdAt: timestamp("created_at", { withTimezone: true, precision: 3 }).defaultNow().notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate( - () => currentTimestamp(), + updatedAt: timestamp("updated_at", { withTimezone: true, precision: 3 }).$onUpdate(() => + currentTimestamp(), ), deletedAt: timestamp("deleted_at", { withTimezone: true, precision: 3 }), }, diff --git a/packages/rpc/src/groups/AnalyticsRpcsDef.ts b/packages/rpc/src/groups/AnalyticsRpcsDef.ts index 4f9b9d41e..41f1fe7b8 100644 --- a/packages/rpc/src/groups/AnalyticsRpcsDef.ts +++ b/packages/rpc/src/groups/AnalyticsRpcsDef.ts @@ -830,6 +830,10 @@ export class AnalyticsRpcsDef extends RpcGroup.make( payload: QueryAnalyticsInsightsRequest, success: QueryAnalyticsInsightsResponse, }), +).middleware(AuthMiddleware) {} + +/** Hosted custom insights, cohorts, and dashboards RPC surface. */ +export class AdvancedAnalyticsRpcsDef extends RpcGroup.make( Rpc.make("QueryCustomAnalyticsInsight", { error: Schema.Union([ RpcActionForbiddenError, diff --git a/packages/rpc/src/index.ts b/packages/rpc/src/index.ts index 3b4080317..cc4a0c494 100644 --- a/packages/rpc/src/index.ts +++ b/packages/rpc/src/index.ts @@ -20,7 +20,6 @@ import { ProductPerkRpcsDef } from "./groups/ProductPerkRpcsDef.ts"; import { ProductRpcsDef } from "./groups/ProductRpcsDef.ts"; import { ProjectRpcsDef } from "./groups/ProjectRpcsDef.ts"; import { UserRpcsDef } from "./groups/UserRpcsDef.ts"; -import { VoidQlRpcsDef } from "./groups/VoidQlRpcsDef.ts"; import { WebhookRpcsDef } from "./groups/WebhookRpcsDef.ts"; import { FeatureFlagRpcsDef } from "./groups/FeatureFlagRpcsDef.ts"; import { FeedbackRpcsDef } from "./groups/FeedbackRpcsDef.ts"; @@ -50,7 +49,6 @@ export const RpcGroups = RpcGroup.make().merge( PaywallRpcsDef, PaywallWorkspaceRpcsDef, UserRpcsDef, - VoidQlRpcsDef, WebhookRpcsDef, ); diff --git a/packages/web-app/src/composition.d.ts b/packages/web-app/src/composition.d.ts index b8eeb56f7..e6547cd4b 100644 --- a/packages/web-app/src/composition.d.ts +++ b/packages/web-app/src/composition.d.ts @@ -18,6 +18,7 @@ declare module "virtual:voidhash-web/auth-server" { declare module "virtual:voidhash-web/edition" { export { + advancedAnalyticsAvailable, isOrganizationWaitlisted, organizationSettingsNavItems, PaywallThumbnailAdminSlot, diff --git a/packages/web-app/src/composition/community/edition.ts b/packages/web-app/src/composition/community/edition.ts index e1ba92989..9d4705f09 100644 --- a/packages/web-app/src/composition/community/edition.ts +++ b/packages/web-app/src/composition/community/edition.ts @@ -10,3 +10,6 @@ export { type OrganizationNavSlotContext, } from "../../features/studio/enterprise/organization-nav-slot"; export { PaywallThumbnailAdminSlot } from "../../features/studio/paywalls/designer/dev-mode/paywall-thumbnail-admin-slot"; + +/** Community exposes only the built-in PostgreSQL analytics pages. */ +export const advancedAnalyticsAvailable = false; diff --git a/packages/web-app/src/features/studio/analytics/custom-dashboards-page.tsx b/packages/web-app/src/features/studio/analytics/custom-dashboards-page.tsx deleted file mode 100644 index 04805d83c..000000000 --- a/packages/web-app/src/features/studio/analytics/custom-dashboards-page.tsx +++ /dev/null @@ -1,1168 +0,0 @@ -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import type { - AnalyticsDashboardType, - QueryCustomAnalyticsInsightResponseType, - SavedAnalyticsInsightType, -} from "@voidhash/rpc"; -import { INTERNAL_FEATURE_FLAGS } from "@voidhash/rpc"; -import { Effect } from "effect"; -import { - Button, - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, - type ChartConfig, - ChartContainer, - Input, - Label, - Page, - PageHeader, - PageHeaderTitle, - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, - Textarea, -} from "@voidhash/ui"; -import { - ArrowDown, - ArrowLeft, - ArrowUp, - ChartSpline, - Columns2, - Copy, - LayoutDashboard, - Pencil, - Plus, - Rows3, - RefreshCw, - SlidersHorizontal, - Save, - Trash2, -} from "lucide-react"; -import { useState } from "react"; -import { Area, AreaChart, Bar, BarChart, Line, LineChart, XAxis, YAxis } from "recharts"; - -import { useAuth } from "@/features/studio/components/auth-context"; -import { - createAnalyticsDashboardOptions, - customAnalyticsInsightQueryOptions, - deleteAnalyticsDashboardOptions, - duplicateAnalyticsDashboardOptions, - listAnalyticsDashboardsOptions, - listAnalyticsInsightsOptions, - listVoidQlInsightsOptions, - putAnalyticsDashboardItemOptions, - queryKeys, - reorderAnalyticsDashboardItemsOptions, - removeAnalyticsDashboardItemOptions, - runSavedVoidQlInsightOptions, - updateAnalyticsDashboardOptions, -} from "@/features/studio/lib/tanstack-query"; -import { useInternalFeatureFlag } from "@/features/studio/lib/useInternalFeatureFlag"; -import { CurrentUser } from "@/features/studio/lib/utils/current-user"; -import { VoidhashErrorCard } from "@/features/studio/shell/components/voidhash-error-card"; - -interface CustomDashboardsPageProps { - organizationSlug: string; - projectSlug: string; -} - -/** Render project dashboards and their saved insight cards. */ -export function CustomDashboardsPage({ organizationSlug, projectSlug }: CustomDashboardsPageProps) { - const { user } = useAuth(); - const project = CurrentUser.getProjectBySlugs(user, organizationSlug, projectSlug); - const queryClient = useQueryClient(); - const [creating, setCreating] = useState(false); - const [selectedDashboardId, setSelectedDashboardId] = useState(); - const [name, setName] = useState("Mobile product overview"); - const [description, setDescription] = useState(""); - const createDashboard = useMutation(createAnalyticsDashboardOptions()); - const dashboards = useQuery({ - ...listAnalyticsDashboardsOptions({ projectId: project?.id ?? "missing-project" }), - enabled: project !== undefined, - }); - - if (!project) { - return ; - } - - const selectedDashboard = dashboards.data?.dashboards.find( - (dashboard) => dashboard.id === selectedDashboardId, - ); - const onCreate = () => { - createDashboard.mutate( - { - description: description.trim() || undefined, - name: name.trim(), - projectId: project.id, - }, - { - onSuccess: async (dashboard) => { - await queryClient.invalidateQueries({ - queryKey: queryKeys.analytics.dashboards({ projectId: project.id }), - }); - setCreating(false); - setDescription(""); - setSelectedDashboardId(dashboard.id); - }, - }, - ); - }; - - if (selectedDashboard) { - return ( - setSelectedDashboardId(undefined)} - organizationId={project.organizationId} - projectId={project.id} - /> - ); - } - - return ( - - setCreating((value) => !value)}> - - New dashboard - - } - > - Dashboards - -
-
-

Analytics dashboards

-

- Arrange reusable insights into focused views for product, growth, and release health. -

-
- - {creating ? ( - - - Create dashboard - - Start empty, then add saved insights as they become useful. - - - -
- - setName(event.target.value)} - value={name} - /> -
-
- -