From d612fbff5be0cc4a53208e983c4d86a655789a21 Mon Sep 17 00:00:00 2001 From: Benjamin Knofe-Vider Date: Sun, 20 Sep 2026 11:52:45 +0200 Subject: [PATCH] feat: manage shared Trino compute pools --- CLAUDE.md | 379 ++ .../migrations/000041_add_trino_pool.sql | 203 + .../configstore/migrations_numbering_test.go | 54 + controlplane/configstore/store.go | 68 +- controlplane/configstore/trino.go | 75 +- controlplane/configstore/trino_pool.go | 293 ++ .../configstore/trino_pool_instances.go | 352 ++ controlplane/configstore/trino_pool_models.go | 333 ++ .../configstore/trino_pool_operations.go | 595 +++ controlplane/multitenant.go | 12 + controlplane/pre_ready_disconnect.go | 3 +- controlplane/pre_ready_disconnect_test.go | 35 + controlplane/provisioner/opa/builder.go | 70 +- controlplane/provisioner/opa/builder_test.go | 109 +- controlplane/provisioner/opa/policy_test.go | 47 + controlplane/provisioner/opa/serve.go | 47 + .../provisioner/trino_cluster_secrets_test.go | 18 + controlplane/provisioner/trino_hoglake.go | 14 + .../trino_projection_fence_test.go | 142 + controlplane/provisioner/trino_provisioner.go | 558 ++- .../provisioner/trino_provisioner_test.go | 84 + controlplane/trino_fleet.go | 12 + controlplane/trino_inputs.go | 15 + controlplane/trino_pool_binding.go | 106 + controlplane/trino_pool_binding_test.go | 132 + controlplane/trino_pool_catalog.go | 532 +++ ...rino_pool_catalog_hoglake_postgres_test.go | 188 + ...ino_pool_catalog_identity_postgres_test.go | 126 + ...trino_pool_catalog_schema_postgres_test.go | 236 ++ .../trino_pool_catalog_unavailable.go | 65 + controlplane/trino_pool_config.go | 256 ++ controlplane/trino_pool_config_source.go | 240 ++ controlplane/trino_pool_config_source_test.go | 239 ++ controlplane/trino_pool_config_test.go | 224 + controlplane/trino_pool_durable.go | 190 + controlplane/trino_pool_effects.go | 532 +++ controlplane/trino_pool_effects_test.go | 258 ++ controlplane/trino_pool_failure.go | 536 +++ .../trino_pool_hoglake_wiring_test.go | 44 + .../trino_pool_member_retries_test.go | 295 ++ controlplane/trino_pool_operator.go | 640 +++ controlplane/trino_pool_operator_test.go | 3616 +++++++++++++++++ controlplane/trino_pool_progress.go | 589 +++ .../trino_pool_projection_authority.go | 215 + .../trino_pool_projection_authority_test.go | 257 ++ controlplane/trino_pool_publication.go | 1379 +++++++ .../trino_pool_publication_order_test.go | 102 + controlplane/trino_pool_receipts.go | 122 + controlplane/trino_pool_validate.go | 582 +++ controlplane/trino_pool_validate_test.go | 510 +++ controlplane/trino_pool_watermark_test.go | 133 + controlplane/trino_pool_wiring.go | 363 ++ controlplane/trino_pool_wiring_test.go | 193 + controlplane/trino_registry.go | 26 +- controlplane/trino_rollout_probe.go | 34 +- controlplane/trinocatalog/publisher.go | 546 +++ controlplane/trinogateway/backend.go | 87 + controlplane/trinogateway/backend_test.go | 89 + controlplane/trinogateway/client.go | 346 ++ controlplane/trinogateway/client_test.go | 296 ++ controlplane/trinogateway/errors.go | 132 + .../testdata/failure_receipt.json | 14 + .../trinogateway/testdata/member.json | 30 + .../trinogateway/testdata/members.json | 30 + .../trinogateway/testdata/obligations.json | 12 + .../testdata/operation_history.json | 14 + .../trinogateway/testdata/pool_state.json | 27 + .../trinogateway/testdata/publication.json | 22 + .../testdata/tenant_admission.json | 13 + controlplane/trinogateway/types.go | 366 ++ controlplane/trinogateway/wire_test.go | 150 + controlplane/trinopool/blueprint.go | 374 ++ controlplane/trinopool/blueprint_test.go | 245 ++ controlplane/trinopool/catalog_version.go | 94 + .../trinopool/catalog_version_test.go | 136 + controlplane/trinopool/instantiate.go | 315 ++ controlplane/trinopool/instantiate_test.go | 301 ++ controlplane/trinopool/phase.go | 124 + controlplane/trinopool/phase_test.go | 141 + controlplane/trinopool/plan.go | 211 + controlplane/trinopool/plan_test.go | 193 + .../trinopool/testdata/blueprint.json | 102 + tests/configstore/helpers_test.go | 21 +- tests/configstore/migrations_postgres_test.go | 90 +- tests/configstore/trino_pool_postgres_test.go | 980 +++++ tests/controlplane/controlplane_test.go | 87 +- tests/mw-dev/README.md | 5 + tests/mw-dev/e2e/harness.sh | 203 + tests/mw-dev/e2e/trino.sh | 16 +- tests/mw-dev/trino_query_diagnostics_test.go | 93 + tests/trinocatalog/publisher_postgres_test.go | 461 +++ .../writer_bridge_postgres_test.go | 211 + tools/gatewaywire/PoolWireFixtures.java | 89 + tools/gatewaywire/README.md | 33 + 94 files changed, 23079 insertions(+), 98 deletions(-) create mode 100644 controlplane/configstore/migrations/000041_add_trino_pool.sql create mode 100644 controlplane/configstore/migrations_numbering_test.go create mode 100644 controlplane/configstore/trino_pool.go create mode 100644 controlplane/configstore/trino_pool_instances.go create mode 100644 controlplane/configstore/trino_pool_models.go create mode 100644 controlplane/configstore/trino_pool_operations.go create mode 100644 controlplane/provisioner/trino_projection_fence_test.go create mode 100644 controlplane/trino_pool_binding.go create mode 100644 controlplane/trino_pool_binding_test.go create mode 100644 controlplane/trino_pool_catalog.go create mode 100644 controlplane/trino_pool_catalog_hoglake_postgres_test.go create mode 100644 controlplane/trino_pool_catalog_identity_postgres_test.go create mode 100644 controlplane/trino_pool_catalog_schema_postgres_test.go create mode 100644 controlplane/trino_pool_catalog_unavailable.go create mode 100644 controlplane/trino_pool_config.go create mode 100644 controlplane/trino_pool_config_source.go create mode 100644 controlplane/trino_pool_config_source_test.go create mode 100644 controlplane/trino_pool_config_test.go create mode 100644 controlplane/trino_pool_durable.go create mode 100644 controlplane/trino_pool_effects.go create mode 100644 controlplane/trino_pool_effects_test.go create mode 100644 controlplane/trino_pool_failure.go create mode 100644 controlplane/trino_pool_hoglake_wiring_test.go create mode 100644 controlplane/trino_pool_member_retries_test.go create mode 100644 controlplane/trino_pool_operator.go create mode 100644 controlplane/trino_pool_operator_test.go create mode 100644 controlplane/trino_pool_progress.go create mode 100644 controlplane/trino_pool_projection_authority.go create mode 100644 controlplane/trino_pool_projection_authority_test.go create mode 100644 controlplane/trino_pool_publication.go create mode 100644 controlplane/trino_pool_publication_order_test.go create mode 100644 controlplane/trino_pool_receipts.go create mode 100644 controlplane/trino_pool_validate.go create mode 100644 controlplane/trino_pool_validate_test.go create mode 100644 controlplane/trino_pool_watermark_test.go create mode 100644 controlplane/trino_pool_wiring.go create mode 100644 controlplane/trino_pool_wiring_test.go create mode 100644 controlplane/trinocatalog/publisher.go create mode 100644 controlplane/trinogateway/backend.go create mode 100644 controlplane/trinogateway/backend_test.go create mode 100644 controlplane/trinogateway/client.go create mode 100644 controlplane/trinogateway/client_test.go create mode 100644 controlplane/trinogateway/errors.go create mode 100644 controlplane/trinogateway/testdata/failure_receipt.json create mode 100644 controlplane/trinogateway/testdata/member.json create mode 100644 controlplane/trinogateway/testdata/members.json create mode 100644 controlplane/trinogateway/testdata/obligations.json create mode 100644 controlplane/trinogateway/testdata/operation_history.json create mode 100644 controlplane/trinogateway/testdata/pool_state.json create mode 100644 controlplane/trinogateway/testdata/publication.json create mode 100644 controlplane/trinogateway/testdata/tenant_admission.json create mode 100644 controlplane/trinogateway/types.go create mode 100644 controlplane/trinogateway/wire_test.go create mode 100644 controlplane/trinopool/blueprint.go create mode 100644 controlplane/trinopool/blueprint_test.go create mode 100644 controlplane/trinopool/catalog_version.go create mode 100644 controlplane/trinopool/catalog_version_test.go create mode 100644 controlplane/trinopool/instantiate.go create mode 100644 controlplane/trinopool/instantiate_test.go create mode 100644 controlplane/trinopool/phase.go create mode 100644 controlplane/trinopool/phase_test.go create mode 100644 controlplane/trinopool/plan.go create mode 100644 controlplane/trinopool/plan_test.go create mode 100644 controlplane/trinopool/testdata/blueprint.json create mode 100644 tests/configstore/trino_pool_postgres_test.go create mode 100644 tests/mw-dev/trino_query_diagnostics_test.go create mode 100644 tests/trinocatalog/publisher_postgres_test.go create mode 100644 tests/trinocatalog/writer_bridge_postgres_test.go create mode 100644 tools/gatewaywire/PoolWireFixtures.java create mode 100644 tools/gatewaywire/README.md diff --git a/CLAUDE.md b/CLAUDE.md index 3eafb6731..4b95cf3ab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1801,6 +1801,385 @@ password/tenant/catalog changes never propagate. the `ui/src/lib/trino.test.ts` derivations and `tests/mw-dev/e2e/trino.sh`. +## Shared Trino Compute Pool (`mode: "shared-pool"`, `kubernetes` tag) — LOAD-BEARING CONTRACT + +Operator-managed pool of immutable Trino instances, replacing fixed blue/green +for a cell that opts in. **Ships disabled**: a cell without `mode: +"shared-pool"` in `DUCKGRES_TRINO_CELLS_FILE` behaves byte for byte as today. +Code: `controlplane/trinopool/` (pure: blueprint, phases, planner, catalog-version +port), `controlplane/trinocatalog/` (fenced catalog publisher), +`controlplane/trinogateway/` (Gateway protocol v1 client), +`controlplane/trino_pool_*.go` (config, effects, validation, operator, wiring, +publication barrier), migration `000041` (upstream owns `000040`, +the Trino backend selection). + +- **Env, all default-off**: `DUCKGRES_TRINO_POOL_ENABLED` (a pooled cell with + this off FAILS startup — falling back to blue/green would hand the operator a + shape they did not ask for), `DUCKGRES_TRINO_POOL_OPERATOR_ENABLED` (off = + desired state is recorded, nothing external is touched), + `DUCKGRES_TRINO_POOL_GATEWAY_URL`, `DUCKGRES_TRINO_POOL_CATALOG_WRITER_ENABLED` + / `_BOOTSTRAP` / `_DSN_FILE` / `_WRITER_IDENTITY`, and + `DUCKGRES_TRINO_POOL_CONFIG_CONFIGMAP` / `_NAMESPACE` / `_REGISTRY_KEY` (the + desired-state source, below — REQUIRED for a pooled cell), and + `DUCKGRES_TRINO_POOL_CATALOG_SCHEMA` (REQUIRED with the catalog writer: the + publisher credential names a database only, and the role's privileges are on + the cell's schema, so unqualified SQL would resolve against `public` where it + can neither create nor read), `DUCKGRES_TRINO_POOL_CATALOG_CELL_ID` (also + REQUIRED with the catalog writer: the catalog store's PARTITION — see the + identity bullet below; there is no default), and + `DUCKGRES_TRINO_POOL_PUBLISHER_IMAGE` (this + process's own digest-pinned image, rendered from the same chart helper as the + container and as the ConfigMap's `publisher-image` key — the projection fence + below compares the two). +- **The pool keeps the cell's identity**: routing group and namespace are + unchanged. Replacing compute never rewrites which warehouse lives where. +- **The catalog store's partition is its OWN identity** + (`DUCKGRES_TRINO_POOL_CATALOG_CELL_ID`, resolved by + `trinoPoolCatalogCellID`): it is the value every coordinator of the cell + carries in `catalog-store.cell-id`, and the rows the publisher writes must + carry exactly it or each coordinator reconciles an EMPTY catalog set. It is + not derived from the pool id — that id carries duckgres's reserved + `registered:` prefix and doubles as the warehouse-ASSIGNMENT key, so deriving + one from the other would force a cluster to spell its partition + `registered:` and would silently re-key every coordinator's read the day + a pool's logical id changes. It is **REQUIRED with the catalog writer and + fails closed** — unset, blank and unusable are one refusal, and there is + deliberately NO fallback to the pool id: a missing setting would otherwise + publish a full catalog set under a partition nobody reads, which is + indistinguishable from a store nothing has been published to yet. The chart + renders it from the same value it renders the coordinators' + `catalog-store.cell-id` from, and guards the pair the way it already guards + the schema pair. The pool row's + `publication_revision` checkpoint addresses the POOL + (`RecordTrinoPoolPublicationRevision` refuses a pool id the held lease does + not name), never the partition. Regressions: + `TestCatalogWriterPublishesUnderTheConfiguredStorePartition` and + `TestCatalogWriterRequiresAStorePartition`. +- **Missing/invalid desired config FREEZES the pool** at last-good state. It is + never a desired count of zero (that would delete the fleet) and never a + startup abort (a ConfigMap problem must not take the CP down). A pool that + vanished from the registry is an error (⇒ freeze), never an empty desired + state. +- **Desired state is published from the API OBJECT, not from the mounted copy + of it.** The registry and blueprint files are the BOOT source — they are what + tells the process a pool exists. Every publication is then derived from ONE + read of `DUCKGRES_TRINO_POOL_CONFIG_CONFIGMAP`, taken immediately before the + write, because a projected volume refreshes per pod on the kubelet's schedule + and a `subPath` mount never refreshes at all: two replicas can hold different + contents indefinitely, and an idle pod can publish what it read at boot. A + pooled cell that does not name that object REFUSES TO START; a silent + fallback would leave nobody able to say which configuration a live pool is + driven from. The blueprint's key is the last element of `blueprint_file` (the + mapping a ConfigMap volume performs), and both documents come from the SAME + read — two reads can straddle an update and pair a new registry entry with a + release the cluster has already replaced. +- **Fences, not leadership.** The janitor lease decides who executes; the + pool's `authority_epoch` fences every durable write — desired-state + publication, freeze and thaw included — every Kubernetes object (annotation, + compared before any adopt) and every Gateway call (`controllerEpoch`). Seeding + the pool row is the ONLY unfenced write and can only INSERT. A refused fenced + write ENDS the leadership term: re-acquiring would let a superseded controller + fence the valid leader straight back, and the two would trade the pool + forever. The fence owner is per-PROCESS, not per control plane. +- **Desired state is ORDERED as well as fenced.** The blueprint carries a + `generation`; a publication that would move it backwards is refused. Holding + the fence proves who may write, not that what they hold is current — a replica + carrying an older blueprint could otherwise publish it over a newer one as a + legal fenced write. The ordinal must come from the config source; a content + hash is not monotonic and would refuse valid configurations. It is a BACKSTOP, + not the freshness mechanism (a settings-only edit need not move it, and two + configurations can carry the same one — hence the API read above), and a + backwards generation FREEZES the pool while KEEPING the lease: it is a + configuration problem, not a lost fence, and reporting it as one ended the + leadership term every tick and handed the pool to a replica that did the same. +- **The catalog writer's fence IS the pool authority.** It is claimed when the + operator wins the lease, never at startup (where every replica would claim it + and the fence would distinguish nobody), and a takeover needs a strictly + higher epoch — an equal epoch held by another identity is a collision, not a + handover. +- **The published catalog revision is read from the catalog STORE, not from + whatever a publication last managed to write down.** The pool row's + `publication_revision` is a cache of it, and the admission gate certifies + members against that number. A catalog can commit while the follow-up write + of its revision fails — and nothing republishes it, because that catalog now + exists, so no later mutation carries the number forward. The gate would keep + certifying members against a revision predating the tenant, admitting it and + reporting the warehouse ready without its catalog. So: a failed checkpoint is + RETURNED from the publish path (not logged and dropped), the writer's claim + checkpoints the revision the takeover already read, and every tenant-admission + tick reconciles the row against the store's own writer state and fails CLOSED + when it cannot. Holding admissions does not hold the compute lifecycle — + `reconcileOnce` isolates that step — so a pool still repairs and drains. +- **A pooled cell answers the managed-Hoglake connector question from the + store.** Adoption of an existing catalog is refused unless its connector is + confirmed (`CatalogConnectors`), which on a coordinator-mediated cell is a + `system.metadata.catalogs` query. A pooled cell has no fixed coordinator, so + the pool writer answers from the `trino_catalogs` rows the coordinators + reconcile FROM. That is the PUBLISHED definition and is sound only for that + decision — whether any member has applied it stays with pool admission, which + proves it per member. Never read a published row as evidence that a catalog is + operational. +- **Failure is not drain.** SUSPECT excludes a member from new work; LOST + requires positive evidence that the admitted process ENDED and is + reported as failed. A repair NAMES the instance it replaces, so the Gateway + charges the repair budget instead of the single planned surge. There is NO + local path back from SUSPECT: the Gateway excluded the member and only a + fresh certified admission un-excludes it, so flipping the row back to SERVING + would claim a member serves while nothing is routed to it. A suspected member + always leaves — proven dead through the loss claim, or replaced through the + planned drain after `trinoPoolSuspectDrainAfter` when it cannot be proven + dead. The Gateway's serving-floor refusal stands either way, so a pool at + its floor keeps the flaky member rather than dropping below it. +- **Two observations prove a process ended, and both are statements Kubernetes + makes after the fact** (`processTerminationEvidence`): every recorded object + is verifiably absent, or **the exact container instance that hosted the + admitted process is no longer the one running in the pod that hosted it**. + The second is what a coordinator that restarts IN PLACE produces: it keeps + every object it had, so absence can never arrive, and the member was retained + until the drain timer — where a drain cannot finish either, because the dead + process's transactions stay pinned to it, holding the instance and its repair + slot forever. + **Exactness is the whole contract.** The container id is recorded at + registration (migration `000041`) precisely so a termination record can be + correlated with the admitted process. An earlier version accepted ANY + termination record on the hosting pod plus a differing identity from the + member's endpoint, and that is NOT evidence: a pod restarted once during + startup carries such a record for its whole life, and a second coordinator + pod makes the endpoint's answer ambiguous about which process replied — + together they declared a live, serving member dead and wrote off its pinned + work (regression: + `TestALiveAdmittedProcessIsNotDeclaredLostByAnUncorrelatedTermination`). + Everything ambiguous fails closed: an unreadable cluster, an unenumerated pod + list, more than one live coordinator pod, a member with no recorded container + id, a termination naming a different instance. A label-filtered listing + omission is NOT deletion (labels are mutable), a timeout is never evidence, + and nothing is deleted or restarted to manufacture proof. + **Evidence boundary, stated honestly:** this rests on what the Kubernetes API + reports. A force-deleted pod object, or a node partitioned from the API + server, can leave a process running that the API no longer describes; neither + observation can see that. Within that boundary "proven dead" means + "Kubernetes reported it dead" — which is what the user accepts, as distinct + from a timeout or routing ambiguity being classified as death. +- **A serving member's PROCESS is observed, not just its pod.** A container can + restart inside a Pod and report ready with the same pod UID and a new Trino + incarnation; the Gateway refuses to dispatch to anything but the boot identity + it registered, so a readiness-only check reports SERVING while that capacity + is gone. The identity is re-read against the admitted one, bounded to one + request per member per `trinoPoolIdentityObserveEvery`. A DIFFERENT identity + suspects the member; an unanswered probe is not evidence and changes nothing; + the stored identity is never quietly updated to match. +- **A FAILED_PREPARING candidate is cleaned up, not abandoned.** It is NOT + terminal: its objects are deleted (sound only because it provably never + admitted work), then its Gateway member is walked PREPARING → SUSPECT → LOST, + which is what releases the pool's live slot, and only then does it become + FAILURE_RETIRED. Leaving it terminal leaked a whole Trino cluster and, at + desired+surge, refused every later registration — no repair, no rollout. The + loss claim carries the coordinator identity the GATEWAY observed at + registration (recorded from the registration response, migration `000041`); + anything re-derived is refused as evidence. +- **Admission is a durable step.** The intent is recorded before the call, so a + lost response is resolved by read-back under the same identity. OK, FAILED + (a decision) and UNKNOWN (no answer) stay distinct. The step identity is the + business intent, never the authority envelope — hashing the whole request made + the retry after a lost response a permanent conflict. A step is recorded + UNKNOWN before the effect and UPDATED with the outcome after it; a decided + outcome is never re-decided. A failed step records `attempts` and + `next_attempt_at` (full jitter, 0.5s→30s) and is not retried until then — the + schedule is durable so a restart cannot reset it to zero — and a step that + succeeds, or an instance that reaches the end of its life, closes its + operation so `terminal_at` is not NULL forever. +- **Instance identity is persisted BEFORE any Kubernetes object exists**, and + names are deterministic, so a lost create is resolved by read-back rather + than by creating a second instance. Identities and live endpoints are never + reused (PK over terminal rows + partial unique index). +- **Blueprint is not a template engine.** Argo delivers a validated JSON with + real `corev1.PodTemplateSpec`s; duckgres injects only object names, labels, + selectors, replica counts and the three env vars the `identity_binding` + declares. Images must be digest-pinned (sidecars included). Each instance + keeps its OWN blueprint snapshot, so a new release cannot change what a + running instance reads. Worker anti-affinity is NARROWED to the instance, not + dropped — a pool-wide selector would make instances fight for nodes. +- **Deletion requires an irreversible Gateway retirement claim** for that exact + incarnation; the phase machine permits it only from `RETIRING` onwards (plus + `FAILED_PREPARING`, which provably never admitted work). Delete verifies each + object's UID itself before passing a precondition, and retirement completes + only on verified absence INCLUDING terminating pods. Pool-shared objects + named in the blueprint are refused by name in both apply and delete. +- **Drain has no deadline.** Sealing is driven by the Gateway's obligations + endpoint; a timer would be a decision to lose open transactions. The serving + floor refusal is surfaced, never overridden. +- **Candidate validation uses no canary.** The candidate is probed through its + OWN Service with the existing observer credential: `/v1/catalog/sync` must be + enabled, ready, zero failed catalogs, applied revision at least the pool's + DURABLE `publication_revision`, a process identity that does not change during + the pass, a worker count that agrees with the running pods, and the pods' + RUNNING images equal to the blueprint's pinned release. The `auth-revision` + check is claimed ONLY when every security component reported what it loaded + AND each projected component reports exactly what THIS control plane is + serving: the OPA bundle's revision (`opa.PolicyRevision`, published in the + bundle as `data.trino.revision` and digesting the POLICY BYTES as well as the + data — `policy.rego` lives in the duckgres binary, so the image check says + nothing about it), and the `sha256:` fingerprints of the projected + `password.db` and `group.db`. Every instance of a required kind must match: a + second password authenticator reads a file duckgres does not write. "The + component answered" is not evidence — a coordinator whose OPA still serves the + projection from before a tenant existed answers perfectly. A coordinator + without `opa.policy.revision-uri` reports OPA as unacknowledged, the check is + absent, and admission fails closed — deliberate, so `opa.policy.revision-uri` + is REQUIRED on a pooled coordinator, pointed at that revision document. +- **The authorization projection is FENCED for a pooled cell.** Every control + plane builds it from its own view and every replica serves it, so two things + are needed together. An ORDER: the config store records the accepted + projection (digest + revision) under the pool's authority, built from source + rows read INSIDE that transaction at `REPEATABLE READ`, with project scopes + derived from the same read rather than the polled snapshot — a revision + allocated for content read at another moment numbers bytes nobody can prove + were current. And an eligible PRODUCER: only a process whose own + `DUCKGRES_TRINO_POOL_PUBLISHER_IMAGE` — captured at STARTUP, from the same + chart helper that renders the container — equals the `publisher-image` key of + the same pool ConfigMap may advance it, compared after the authority is held. + The startup value is used rather than the pod's spec or a runtime image ID: a + pod specification can be edited under a running process, and the runtime + identity is the node's platform-specific digest rather than the manifest the + chart names. Both sides must be pinned by `@sha256:<64 hex>`; a floating tag + on either side is refused, because two equal tags are not evidence of equal + bytes. That is equality, never an ordering + of image identities — an older binary that wins the lease would otherwise + publish its own older `policy.rego` under a HIGHER revision, which no counter + can detect, and a deliberate rollback moves the desired value so the older + binary becomes eligible again. Every uncertainty fails closed, so a partial + rollout PAUSES pooled publication (OPA keeps its last-good bundle) rather than + regressing it. The auth Secret is written under that revision with the + object's own resourceVersion as the CAS and refuses to replace a newer stamp; + the bundle handler gates the bundle it CAPTURED before a 200 AND before a 304 + (a 304 preserves exactly the stale bundle the fence exists to retire), and an + unreadable record is a refusal. That gate reads the durable record on EVERY + request and must stay uncached: OPA's periodic downloader activates each + bundle inline as it fetches it, so a cached "still accepted" answer does not + merely delay the truth, it INSTALLS a superseded projection after a newer one + was accepted and a tenant admitted against it. A replica that is not the + advancer still BUILDS and SERVES its projection — refusing would strand every + coordinator polling it on its last-good bundle — but writes the auth Secret + only when what it built equals the accepted digest, and being un-advanceable + is never reported as a reconcile failure (every replica but one is in that + state at any moment, and failing would mark every pooled warehouse Failed). + Candidate admission compares against the + accepted projection, not against what the local process last published. + Legacy cells install no fence and are byte-for-byte unchanged. +- **A retried step repeats the IDENTICAL request — every field, not just the + generation.** The Gateway journals each step under its identity and hashes + the WHOLE request body, minus the authority envelope, so a step whose effect + landed while its response was lost can only be resolved by repeating exactly + what was sent. Anything rebuilt from freshly read state reports a changed + intent, and that member can never finish the transition — not on the next + tick, not after a leader change, never. Three fields did that: the expected + generation (sealing read it back from the obligations the seal had just + moved), the loss claim's `observedAt` (re-stamped per attempt; it now carries + the durable moment the member was excluded), and registration's boot identity + (re-probed per attempt, so an in-place restart changed it). Where the request + genuinely cannot be reproduced — registration's observed identity, a + suspicion whose reason depends on which check fired first — the member is + READ BACK and what the Gateway recorded is adopted, never re-derived. A + read-back that FAILS stays an error: an unresolved operation identity is not + permission to send a body this controller derived meanwhile. The accepted + read-back states are explicit SETS, not a rank — the lifecycle is a graph + (SUSPECT ↔ DRAINING), and a retirement counts as a recorded loss only when + its kind is FAILED, because a DRAINED one completed a different transition. + `observedAt` is omitted from the loss claim: it is optional, the Gateway + stamps its own receipt time, and every value available here is either + re-derived per attempt or not the termination-observation time at all. + Regressions: `TestALostResponseDoesNotChangeTheRetriedRequest` and + `TestALostResponseDoesNotChangeTheRetriedMemberRequest`, against a fake that + journals the whole payload the way the Gateway does. +- **One instance never stops the pool.** `progressInstances` records a failing + instance and carries on, and the planner still runs, so a member that cannot + make progress no longer holds every repair, drain and replacement behind it. + Only a lost fence ends the sweep. This is the isolation the tenant loop + already applies. +- **Tenant admission is a committed publication, not a published binding.** + Publishing principals (PUT `…/tenants/{t}/principals`) tells the Gateway which + logins belong to a tenant; the Gateway dispatches work only for an ADMITTED + one, and only the barrier (open → receipt per active member → commit) admits + it. A receipt is EVIDENCE: each member's own `/v1/catalog/sync` must report + the published catalog revision applied and all three projection revisions + current before duckgres records it. A tenant that leaves the projection is + REVOKED, once (the durable row is kept); with the gate on, a warehouse is held + at Provisioning until its publication commits. Readiness then follows the + COMMITTED target revision, not the state field: a tenant that adds a login + opens a new attempt, and reading the state alone would flap a warehouse that + has served for weeks back to Provisioning because somebody created a user. + All of it reads the durable record (migration `000041`), never the + leader's memory. +- **A refusal settles a tenant's occurrence only on the FIRST request of it.** + The Gateway rolls a refused call back before it journals anything, so the step + identity stays unclaimed, and publishing principals carries no revision + ordering of its own. On a REISSUE an earlier copy of the same request may + still be executing there, so closing the occurrence would let the controller + advance to a newer intent, publish and checkpoint it, and then have that older + copy land and overwrite the Gateway's binding with the superseded principal + set — which duckgres never corrects, because it believes it already published + the newer one. A removed login would stay dispatchable and a current one could + not dispatch. `failTenantStep` therefore takes `firstAttempt`: true from + `publishChangedBindings`/`revokeDepartedTenant`, false from + `reissuePublication`/`reissueRevoke`. The property the tests pin is that what + duckgres has checkpointed always names the set the Gateway actually binds + (`TestACheckpointedBindingNeverDivergesFromTheGateway`), NOT that the newest + desired set wins. + **Known limitation (followup, not recovery):** a tenant whose first request + was UNKNOWN and whose reissue is then refused for a condition that never + clears stays HELD on that occurrence — it publishes no corrected binding and + is not revoked until the refusal stops. That is the conservative direction + (the alternative loses a binding silently), and resolving it needs either a + durable "was this occurrence ever unknown" bit or a Gateway way to burn a step + identity. Both are out of scope here; do not "fix" it by closing the + occurrence on a reissue. +- **One authoritative boot identity.** The coordinator's `processId` is probed + BEFORE member registration and is sent as `bootId` on both register and admit; + the Gateway requires the receipt to carry the pair it recorded. A restart + before admission fails the candidate (`FAILED_PREPARING`) rather than retrying + against an incarnation that no longer exists. +- **Transport is plain internal HTTP** to each instance's ClusterIP Service; TLS + terminates at the Gateway. Probes declare the forwarded HTTPS hop + (`X-Forwarded-Proto`) rather than relaxing auth, and + `allow-insecure-over-http` is never set. There is no pool certificate, no + private CA and no `tls_server_name` for a pooled instance. Credentials and + query data cross the cluster network unencrypted on this hop — accepted, with + coordinator NetworkPolicies a deferred follow-up, so pool admission must NOT + be described as non-bypassable by an in-cluster caller. +- **The Gateway client matches the Java source, not a design doc** + (`GET members` is a bare array; admission is one `admit` call with a nested + receipt; fields are `desiredMembers`/`maxRepair`; the Gateway computes the + guard payload hash itself). Decoding is pinned by fixtures generated from the + real records — regenerate with `tools/gatewaywire`. Pooled registration needs + a Gateway backend record, created INACTIVE and never activated through the + legacy route. +- **Principal binding is duckgres-authoritative.** The tenant's principal set + is derived from the SAME projection that writes `password.db`, asserted + identical in test; the gate would otherwise block real users or admit + principals Trino rejects. +- **Catalog writer**: one fenced transaction per mutation against the + Trino-side catalog store's schema (writer-state row lock, exact epoch AND + identity, journal replay resolution, `catalog_count` recomputed inside the + transaction; the tables are created by `trinocatalog.EnsureSchema` and + asserted against a real PostgreSQL in `tests/trinocatalog/`). + Takeover is explicit; a mutation never claims a higher epoch implicitly. A + lost COMMIT is resolved from the journal, never retried blind. +- Touching any of this → update `controlplane/trinopool/*_test.go`, + `controlplane/trinogateway/*_test.go` (fixtures are generated from the real + Java records — see `tools/gatewaywire/README.md`), + `controlplane/trino_pool_*_test.go`, + `controlplane/trino_pool_member_retries_test.go`, + `controlplane/trino_pool_publication_order_test.go`, + `tests/configstore/trino_pool_postgres_test.go`, + `tests/trinocatalog/*_postgres_test.go`, AND the + `trino_shared_pool_disabled` assertion in `tests/mw-dev/e2e/harness.sh`. + The pool's migrations are numbered ABOVE upstream's, so adding one means + extending `TestMigrationVersionsAreUnique`'s range implicitly (it reads the + embedded directory) and updating the version asserts in + `tests/configstore/migrations_postgres_test.go` — including + `TestConfigStoreMigration40PinsExistingTrinoBackends`, which rewinds every + version at or above 40 so goose can re-apply them in order. + ## Logical Catalog Alias (`org_` as the startup `database`) A pgwire session may select its catalog by the name the org has on Trino diff --git a/controlplane/configstore/migrations/000041_add_trino_pool.sql b/controlplane/configstore/migrations/000041_add_trino_pool.sql new file mode 100644 index 000000000..3faaf90cc --- /dev/null +++ b/controlplane/configstore/migrations/000041_add_trino_pool.sql @@ -0,0 +1,203 @@ +-- +goose Up +-- Durable state for the shared Trino compute pool (the operator side of the +-- shared-pool work). Every table here is inert while the feature flags are off: +-- nothing reads or writes them unless a registered cell declares +-- `mode: "shared-pool"` AND DUCKGRES_TRINO_POOL_ENABLED is set. +-- +-- Why these live in duckgres rather than in the Gateway database: duckgres owns +-- the DESIRED pool specification, the immutable instance identities and the +-- durable reconcile operations. The Gateway owns admission and the irreversible +-- retirement claim. There is no distributed transaction between the two; the +-- ordering is durable-intent-first, external-effect-second, read-back on any +-- unknown outcome. The receipt columns below are where a Gateway outcome is +-- checkpointed after it has been read back. + +CREATE TABLE duckgres_trino_pools ( + pool_id TEXT PRIMARY KEY, + public_id TEXT NOT NULL, + api_mode TEXT NOT NULL DEFAULT 'legacy', + desired_release_id TEXT NOT NULL DEFAULT '', + desired_blueprint_digest TEXT NOT NULL DEFAULT '', + -- Sizing. desired_instances is deliberately NOT NULL with no default of + -- zero-meaning-empty: a missing or unreadable desired configuration sets + -- `frozen` instead. "Desired count zero" must never be the way a config + -- problem expresses itself, or a bad mount deletes the fleet. + desired_instances INTEGER NOT NULL DEFAULT 3 CHECK (desired_instances >= 1), + min_serving INTEGER NOT NULL DEFAULT 3 CHECK (min_serving >= 1), + max_surge INTEGER NOT NULL DEFAULT 1 CHECK (max_surge >= 0), + max_repair INTEGER NOT NULL DEFAULT 1 CHECK (max_repair >= 0), + -- Monotonic authority epoch. Bumped on leader takeover, sent to the Gateway + -- as controllerEpoch and to the catalog store as writer_epoch, so one + -- number fences every external effect of this pool. + authority_epoch BIGINT NOT NULL DEFAULT 0 CHECK (authority_epoch >= 0), + authority_owner TEXT NOT NULL DEFAULT '', + -- Desired and admitted configuration revisions are separate values on + -- purpose: advancing desired state must not instantly make every existing + -- coordinator ineligible. + publication_revision BIGINT NOT NULL DEFAULT 0 CHECK (publication_revision >= 0), + admitted_revision BIGINT NOT NULL DEFAULT 0 CHECK (admitted_revision >= 0), + frozen BOOLEAN NOT NULL DEFAULT FALSE, + frozen_reason TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Order desired configuration independently of the controller's authority epoch. + desired_generation BIGINT NOT NULL DEFAULT 0 CHECK (desired_generation >= 0) +); + +CREATE TABLE duckgres_trino_pool_instances ( + -- Never reused, for the lifetime of the pool: a retired identity coming + -- back would let a stale Gateway or Kubernetes reference resolve to a live + -- instance. The primary key covers retired rows too, which is what enforces + -- it. + instance_id TEXT PRIMARY KEY, + pool_id TEXT NOT NULL REFERENCES duckgres_trino_pools (pool_id) ON DELETE CASCADE, + release_id TEXT NOT NULL, + spec_digest TEXT NOT NULL, + -- The instance's own immutable copy of the blueprint. Argo may replace or + -- prune the source ConfigMap during a new release; a PREPARING, SERVING or + -- DRAINING instance keeps the configuration it was created with. + blueprint_snapshot JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(blueprint_snapshot) = 'object'), + phase TEXT NOT NULL DEFAULT 'PENDING', + phase_changed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + owner_epoch BIGINT NOT NULL DEFAULT 0, + -- Charged to the failure-repair budget rather than the planned surge. + repair BOOLEAN NOT NULL DEFAULT FALSE, + -- Kubernetes inventory. UIDs are recorded so a delete can carry a + -- precondition: a name alone could match an object somebody else recreated. + coordinator_deployment_name TEXT NOT NULL DEFAULT '', + coordinator_deployment_uid TEXT NOT NULL DEFAULT '', + worker_deployment_name TEXT NOT NULL DEFAULT '', + worker_deployment_uid TEXT NOT NULL DEFAULT '', + service_name TEXT NOT NULL DEFAULT '', + service_uid TEXT NOT NULL DEFAULT '', + config_map_name TEXT NOT NULL DEFAULT '', + config_map_uid TEXT NOT NULL DEFAULT '', + coordinator_pod_uid TEXT NOT NULL DEFAULT '', + -- Process identity observed from the candidate itself, never asserted. + coordinator_node_id TEXT NOT NULL DEFAULT '', + coordinator_boot_id TEXT NOT NULL DEFAULT '', + endpoint_url TEXT NOT NULL DEFAULT '', + tls_server_name TEXT NOT NULL DEFAULT '', + -- Gateway's view, checkpointed after read-back. Gateway stays authoritative. + gateway_incarnation TEXT NOT NULL DEFAULT '', + gateway_backend_name TEXT NOT NULL DEFAULT '', + gateway_state TEXT NOT NULL DEFAULT '', + gateway_generation BIGINT NOT NULL DEFAULT 0, + applied_catalog_revision BIGINT NOT NULL DEFAULT 0, + validation_receipt JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(validation_receipt) = 'object'), + validated_at TIMESTAMPTZ NULL, + retirement_receipt JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(retirement_receipt) = 'object'), + last_error TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Record both ConfigMap identities for UID-preconditioned deletion. + worker_config_map_name TEXT NOT NULL DEFAULT '', + worker_config_map_uid TEXT NOT NULL DEFAULT '', + -- Name the failed member so the Gateway charges the repair budget. + repair_for TEXT NOT NULL DEFAULT '', + failure_reason TEXT NOT NULL DEFAULT '', + -- Preserve the exact coordinator and container identities admitted by the Gateway. + coordinator_id TEXT NOT NULL DEFAULT '', + coordinator_container_id TEXT NOT NULL DEFAULT '' +); + +-- The reconcile loop's only listing query is "every instance of this pool". +CREATE INDEX idx_duckgres_trino_pool_instances_pool + ON duckgres_trino_pool_instances (pool_id, phase); + +-- A pooled endpoint is never reused by a second instance, mirroring the same +-- rule on the Gateway side. Partial so retired rows keep their history without +-- reserving the address forever. +CREATE UNIQUE INDEX idx_duckgres_trino_pool_instances_endpoint + ON duckgres_trino_pool_instances (pool_id, endpoint_url) + WHERE endpoint_url <> '' AND phase NOT IN ('RETIRED', 'FAILURE_RETIRED', 'FAILED_PREPARING'); + +CREATE TABLE duckgres_trino_pool_operations ( + operation_id TEXT PRIMARY KEY, + pool_id TEXT NOT NULL REFERENCES duckgres_trino_pools (pool_id) ON DELETE CASCADE, + instance_id TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + -- The immutable intent. The same operation id with a different hash is a + -- conflict, never an overwrite: that is what makes a lost response safe to + -- resolve by read-back instead of by inventing a new operation. + intent_hash TEXT NOT NULL, + owner_epoch BIGINT NOT NULL DEFAULT 0, + step TEXT NOT NULL DEFAULT '', + phase TEXT NOT NULL DEFAULT 'pending', + receipts JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(receipts) = 'object'), + last_error TEXT NOT NULL DEFAULT '', + attempts BIGINT NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + terminal_at TIMESTAMPTZ NULL +); + +CREATE INDEX idx_duckgres_trino_pool_operations_open + ON duckgres_trino_pool_operations (pool_id, phase) + WHERE terminal_at IS NULL; + +-- Per-step idempotency. Scoping replay identity to the step as well as the +-- parent operation is what lets a resumed operation re-run only the step that +-- was interrupted. +CREATE TABLE duckgres_trino_pool_operation_steps ( + operation_id TEXT NOT NULL REFERENCES duckgres_trino_pool_operations (operation_id) ON DELETE CASCADE, + step_id TEXT NOT NULL, + payload_hash TEXT NOT NULL, + outcome TEXT NOT NULL DEFAULT 'pending', + result JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(result) = 'object'), + recorded_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (operation_id, step_id) +); + +-- Per-warehouse publication. Desired, published and admitted are three distinct +-- facts and are stored as three distinct values; overloading one boolean is how +-- a half-published tenant ends up looking ready. +CREATE TABLE duckgres_trino_pool_publications ( + pool_id TEXT NOT NULL REFERENCES duckgres_trino_pools (pool_id) ON DELETE CASCADE, + org_id TEXT NOT NULL, + desired_revision BIGINT NOT NULL DEFAULT 0, + published_revision BIGINT NOT NULL DEFAULT 0, + admitted_revision BIGINT NOT NULL DEFAULT 0, + publication_id TEXT NOT NULL DEFAULT '', + publication_operation_id TEXT NOT NULL DEFAULT '', + state TEXT NOT NULL DEFAULT 'pending', + gateway_receipt JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(gateway_receipt) = 'object'), + last_error TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Barrier revisions identify principal bindings and acknowledged configuration. + principal_revision TEXT NOT NULL DEFAULT '', + target_revision TEXT NOT NULL DEFAULT '', + admitted_target_revision TEXT NOT NULL DEFAULT '', + -- Persist occurrence identities and retry schedules across controller restarts. + attempt BIGINT NOT NULL DEFAULT 0 CHECK (attempt >= 0), + attempts BIGINT NOT NULL DEFAULT 0 CHECK (attempts >= 0), + next_attempt_at TIMESTAMPTZ NULL, + -- Replay unknown outcomes with the original intent and identical payload. + -- Payloads contain principal identifiers and revisions, never credentials. + pending_intent TEXT NOT NULL DEFAULT '' CHECK (pending_intent IN ('', 'principals', 'revoke')), + pending_payload JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(pending_payload) = 'object'), + PRIMARY KEY (pool_id, org_id) +); + +-- The authorization-projection watermark. A duckgres replica consults this +-- before serving an OPA bundle and refuses to serve anything older, so a stale +-- replica never emits a regressing bundle in the first place. An ETag on the +-- producer alone cannot do that: it cannot reject a response that is already in +-- flight, and stock OPA does not compare custom bundle revisions. +CREATE TABLE duckgres_trino_pool_projection ( + pool_id TEXT PRIMARY KEY REFERENCES duckgres_trino_pools (pool_id) ON DELETE CASCADE, + authority_epoch BIGINT NOT NULL DEFAULT 0 CHECK (authority_epoch >= 0), + accepted_revision BIGINT NOT NULL DEFAULT 0 CHECK (accepted_revision >= 0), + accepted_digest TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- +goose Down +DROP TABLE IF EXISTS duckgres_trino_pool_projection; +DROP TABLE IF EXISTS duckgres_trino_pool_publications; +DROP TABLE IF EXISTS duckgres_trino_pool_operation_steps; +DROP TABLE IF EXISTS duckgres_trino_pool_operations; +DROP TABLE IF EXISTS duckgres_trino_pool_instances; +DROP TABLE IF EXISTS duckgres_trino_pools; diff --git a/controlplane/configstore/migrations_numbering_test.go b/controlplane/configstore/migrations_numbering_test.go new file mode 100644 index 000000000..5fbfc7d0d --- /dev/null +++ b/controlplane/configstore/migrations_numbering_test.go @@ -0,0 +1,54 @@ +package configstore + +import ( + "strconv" + "strings" + "testing" +) + +// Every migration owns its own version number. +// +// goose keys applied migrations by that number, so two files sharing one is not +// a cosmetic clash: whichever ran first records the version, and the second is +// then considered already applied and NEVER runs. The schema it was supposed to +// create is simply absent, on every environment, with no error anywhere. +// +// This is exactly what a long-lived branch produces when it is merged: two +// sides each add "the next" migration. The tripwire is cheap and the failure it +// prevents is silent. +func TestMigrationVersionsAreUnique(t *testing.T) { + entries, err := configStoreMigrationFS.ReadDir("migrations") + if err != nil { + t.Fatalf("read migrations: %v", err) + } + owner := map[int64]string{} + var highest int64 + for _, entry := range entries { + name := entry.Name() + if !strings.HasSuffix(name, ".sql") { + continue + } + digits, _, found := strings.Cut(name, "_") + if !found { + t.Fatalf("migration %q is not _.sql", name) + } + version, err := strconv.ParseInt(digits, 10, 64) + if err != nil || version < 1 { + t.Fatalf("migration %q has no usable version prefix", name) + } + if previous, taken := owner[version]; taken { + t.Fatalf("migrations %q and %q share version %d; one of them would never run", previous, name, version) + } + owner[version] = name + if version > highest { + highest = version + } + } + // No gaps either: a hole is how two branches end up "renumbered" onto the + // same free slot later. + for version := int64(1); version <= highest; version++ { + if _, present := owner[version]; !present { + t.Fatalf("migration version %d is missing; the sequence has a hole up to %d", version, highest) + } + } +} diff --git a/controlplane/configstore/store.go b/controlplane/configstore/store.go index cd1d077fd..49a4930d7 100644 --- a/controlplane/configstore/store.go +++ b/controlplane/configstore/store.go @@ -427,37 +427,55 @@ func orgUserQueryAccessFromSnapshot(snapshot *Snapshot, orgID, username string) return policy, true } for _, team := range org.Teams { - if team.TeamID != *access.TeamID || !team.Enabled { + if team.TeamID != *access.TeamID { continue } - policy.ReadOnly = access.Mode != OrgUserAccessModeProjectUser - importsSchema := team.SchemaName + "_data_imports" - if team.SchemaDataImportsName != nil && *team.SchemaDataImportsName != "" { - importsSchema = *team.SchemaDataImportsName - } - policy.AllowedSchemas = []string{ - importsSchema, - fmt.Sprintf("shadow_%d_models", team.TeamID), - team.SchemaName, - } - // A non-NULL override means the team's table lives in the shared - // legacy posthog schema — even when the override spells the derived - // default name (posthog org team 2 is events_table_name="events" → - // posthog.events). NULL means derive from schema_name, which the - // AllowedSchemas grant above already covers. - if team.EventsTableName != nil && *team.EventsTableName != "" { - policy.AllowedRelations = append(policy.AllowedRelations, "posthog."+*team.EventsTableName) - } - if team.PersonsTableName != nil && *team.PersonsTableName != "" { - policy.AllowedRelations = append(policy.AllowedRelations, "posthog."+*team.PersonsTableName) - } - sort.Strings(policy.AllowedSchemas) - sort.Strings(policy.AllowedRelations) - return policy, true + return OrgUserQueryAccessForTeam(access.Mode, team), true } return policy, true } +// OrgUserQueryAccessForTeam derives a project-scoped login's policy from ITS +// TEAM ROW. +// +// It is the one derivation, shared by the snapshot-backed reader above and by +// the pooled projection path that reads the same row inside its own +// transaction. Two implementations would be two answers to "what may this login +// see", and the pgwire gateway and the Trino bundle would eventually disagree. +// +// A disabled team resolves to the fail-closed policy: no namespaces and no +// write authorization, whatever the mode says. +func OrgUserQueryAccessForTeam(mode string, team OrgTeamConfig) OrgUserQueryAccess { + policy := OrgUserQueryAccess{ReadOnly: true} + if !team.Enabled { + return policy + } + policy.ReadOnly = mode != OrgUserAccessModeProjectUser + importsSchema := team.SchemaName + "_data_imports" + if team.SchemaDataImportsName != nil && *team.SchemaDataImportsName != "" { + importsSchema = *team.SchemaDataImportsName + } + policy.AllowedSchemas = []string{ + importsSchema, + fmt.Sprintf("shadow_%d_models", team.TeamID), + team.SchemaName, + } + // A non-NULL override means the team's table lives in the shared + // legacy posthog schema — even when the override spells the derived + // default name (posthog org team 2 is events_table_name="events" → + // posthog.events). NULL means derive from schema_name, which the + // AllowedSchemas grant above already covers. + if team.EventsTableName != nil && *team.EventsTableName != "" { + policy.AllowedRelations = append(policy.AllowedRelations, "posthog."+*team.EventsTableName) + } + if team.PersonsTableName != nil && *team.PersonsTableName != "" { + policy.AllowedRelations = append(policy.AllowedRelations, "posthog."+*team.PersonsTableName) + } + sort.Strings(policy.AllowedSchemas) + sort.Strings(policy.AllowedRelations) + return policy +} + // Snapshot returns the current config snapshot. func (cs *ConfigStore) Snapshot() *Snapshot { cs.mu.RLock() diff --git a/controlplane/configstore/trino.go b/controlplane/configstore/trino.go index 8570fd68d..db8ccdf87 100644 --- a/controlplane/configstore/trino.go +++ b/controlplane/configstore/trino.go @@ -326,12 +326,40 @@ func (cs *ConfigStore) DisableTrino(orgID string) error { // filters by cell (see TrinoEnabledOrg.CellID for why the filter is not in // the SQL). func (cs *ConfigStore) ListTrinoEnabledOrgs() ([]TrinoEnabledOrg, error) { + return cs.listTrinoEnabledOrgs(cs.db) +} + +// listTrinoEnabledOrgs reads the projection's source rows through the given +// handle. +// +// It takes a handle rather than using cs.db so a caller can read it INSIDE the +// transaction that allocates the projection's accepted revision. That coupling +// is the point: a revision allocated for content read at some other moment +// describes bytes nobody can prove were current, which is exactly how a stale +// buffer ends up stamped with a fresh number. +func (cs *ConfigStore) listTrinoEnabledOrgs(db *gorm.DB) ([]TrinoEnabledOrg, error) { + return cs.listTrinoEnabledOrgsWith(db, cs.snapshotScopeResolver()) +} + +// listTrinoEnabledOrgsCoherently reads the projection's source rows AND each +// scoped login's team row through one handle, so every part of the result comes +// from the same view of the database. +// +// The snapshot-backed resolver cannot be used here: it is refreshed on a poll, +// so a scope read from it can be older than the rows this transaction just +// read - and a projection built from that mixture would be accepted as one +// coherent thing. +func (cs *ConfigStore) listTrinoEnabledOrgsCoherently(db *gorm.DB) ([]TrinoEnabledOrg, error) { + return cs.listTrinoEnabledOrgsWith(db, transactionScopeResolver(db)) +} + +func (cs *ConfigStore) listTrinoEnabledOrgsWith(db *gorm.DB, scope trinoScopeResolver) ([]TrinoEnabledOrg, error) { var out []TrinoEnabledOrg // Inner join with duckgres_org_users on (org_id, username='root') so a // missing OrgUser row drops the org from the result. Inner join with // duckgres_orgs for database_name, which is the org's Trino principal — // a missing or blank one drops the org for the same reason. - err := cs.db.Table("duckgres_managed_warehouse_trino AS t"). + err := db.Table("duckgres_managed_warehouse_trino AS t"). Select(`t.org_id AS org_id, o.database_name AS database_name, t.tier AS tier, @@ -353,7 +381,7 @@ func (cs *ConfigStore) ListTrinoEnabledOrgs() ([]TrinoEnabledOrg, error) { if len(out) == 0 { return out, nil } - if err := cs.attachTrinoOrgUsers(out); err != nil { + if err := cs.attachTrinoOrgUsersWith(db, out, scope); err != nil { return nil, err } return out, nil @@ -395,13 +423,50 @@ type trinoOrgUserRow struct { // project login may read. A scoped row whose scope cannot be resolved is // dropped rather than projected unscoped: an unresolvable scope must never // silently widen into org-wide access. -func (cs *ConfigStore) attachTrinoOrgUsers(orgs []TrinoEnabledOrg) error { +// trinoScopeResolver answers "what may this project-scoped login see". +// +// The snapshot-backed resolver is the legacy behavior and stays the default. +// The pooled acceptance path passes a resolver that reads the team row from ITS +// OWN transaction instead: the in-memory snapshot is refreshed on a poll, so a +// scope read from it can be older than the rows the same transaction just read, +// and a projection accepted on that basis would be stamped as coherent while +// mixing two ages of source data. +type trinoScopeResolver func(orgID, username, mode string, teamID *int64) (OrgUserQueryAccess, bool) + +func (cs *ConfigStore) snapshotScopeResolver() trinoScopeResolver { + return func(orgID, username, _ string, _ *int64) (OrgUserQueryAccess, bool) { + return cs.OrgUserQueryAccess(orgID, username) + } +} + +// transactionScopeResolver derives each scoped login's policy from the team +// rows visible to THIS transaction. +func transactionScopeResolver(db *gorm.DB) trinoScopeResolver { + return func(orgID, _, mode string, teamID *int64) (OrgUserQueryAccess, bool) { + if teamID == nil { + return OrgUserQueryAccess{}, false + } + var team OrgTeamConfig + err := db.Table("duckgres_org_teams"). + Select("team_id, schema_name, enabled, events_table_name, persons_table_name, schema_data_imports_name"). + Where("org_id = ? AND team_id = ?", orgID, *teamID). + Scan(&team).Error + if err != nil || team.TeamID != *teamID { + // No row, or the read failed: the scope is unresolvable, and an + // unresolvable scope must never widen into org-wide access. + return OrgUserQueryAccess{}, false + } + return OrgUserQueryAccessForTeam(mode, team), true + } +} + +func (cs *ConfigStore) attachTrinoOrgUsersWith(db *gorm.DB, orgs []TrinoEnabledOrg, scope trinoScopeResolver) error { ids := make([]string, 0, len(orgs)) for _, o := range orgs { ids = append(ids, o.OrgID) } var rows []trinoOrgUserRow - err := cs.db.Table("duckgres_org_users"). + err := db.Table("duckgres_org_users"). Select("org_id, username, password, access_mode, team_id"). Where("org_id IN ?", ids). Where("disabled = ?", false). @@ -416,7 +481,7 @@ func (cs *ConfigStore) attachTrinoOrgUsers(orgs []TrinoEnabledOrg) error { for _, r := range rows { u := TrinoOrgUser{Username: r.Username, PasswordHash: r.Password} if IsProjectScopedAccessMode(r.AccessMode) { - access, scoped := cs.OrgUserQueryAccess(r.OrgID, r.Username) + access, scoped := scope(r.OrgID, r.Username, r.AccessMode, r.TeamID) if !scoped || r.TeamID == nil { // The row says scoped but the snapshot does not agree -- // an unloaded or stale snapshot, or a user written since diff --git a/controlplane/configstore/trino_pool.go b/controlplane/configstore/trino_pool.go new file mode 100644 index 000000000..e23de9cd1 --- /dev/null +++ b/controlplane/configstore/trino_pool.go @@ -0,0 +1,293 @@ +package configstore + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// Fenced access to the shared Trino pool state. +// +// Two rules hold everywhere in this file: +// +// - Leader election coordinates EXECUTION; the database fences EFFECTS. Every +// write carries a TrinoPoolLease and is refused if the pool's stored epoch +// has moved on, so a superseded leader that has not noticed yet cannot +// write on top of its successor. +// - A conflict is never resolved by re-reading a fresher epoch. Losing the +// CAS means losing authority; the caller stops, it does not retry harder. + +// UpsertTrinoPoolSpec applies the desired configuration resolved from the +// registry and blueprint. It touches only desired fields; the authority epoch, +// the freeze flag and every runtime column belong to the operator. +// +// It is FENCED. Publishing desired state changes what the +// operator will do next, so it is a lifecycle mutation like any other: a +// delayed old leader, or a replica still holding stale configuration, must not +// be able to overwrite a newer desired spec. +// +// The pool row must already exist for a fenced write to be possible, so the +// first publication seeds it - see SeedTrinoPool. +func (cs *ConfigStore) UpsertTrinoPoolSpec(ctx context.Context, lease TrinoPoolLease, spec TrinoPoolSpec) error { + if err := spec.validate(); err != nil { + return err + } + if spec.PoolID != lease.PoolID { + return fmt.Errorf("%w: spec is for pool %q, lease covers %q", ErrTrinoPoolConflict, spec.PoolID, lease.PoolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, pool *TrinoPool) error { + // Stale CONTENT is refused even from a valid leader. Holding the fence + // proves who may write, not that what they hold is current. + // + // This orders GENERATIONS, and only generations. It does not by itself + // establish that a leader's configuration is current: two different + // configurations can carry the same generation, and whatever supplies + // the value may not change it for a settings-only edit. Freshness comes + // from re-reading the authoritative configuration at the moment of the + // write (see the operator's per-tick resolution); this check is the + // backstop against a value that provably went backwards. + if spec.Generation < pool.DesiredGeneration { + return fmt.Errorf("%w: %d is behind the published %d", + ErrTrinoPoolStaleGeneration, spec.Generation, pool.DesiredGeneration) + } + return tx.Model(&TrinoPool{}).Where("pool_id = ?", spec.PoolID).Updates(map[string]any{ + "desired_generation": spec.Generation, + "public_id": spec.PublicID, + "api_mode": spec.APIMode, + "desired_release_id": spec.DesiredReleaseID, + "desired_blueprint_digest": spec.DesiredBlueprintDigest, + "desired_instances": spec.DesiredInstances, + "min_serving": spec.MinServing, + "max_surge": spec.MaxSurge, + "max_repair": spec.MaxRepair, + "updated_at": time.Now().UTC(), + }).Error + }) +} + +// SeedTrinoPool creates the pool row if it does not exist yet. It is the one +// unfenced write, because a fence needs a row to lock: it only ever INSERTs, +// never updates, so it cannot overwrite anything another leader published. +func (cs *ConfigStore) SeedTrinoPool(ctx context.Context, spec TrinoPoolSpec) error { + if err := spec.validate(); err != nil { + return err + } + pool := TrinoPool{ + DesiredGeneration: spec.Generation, + PoolID: spec.PoolID, + PublicID: spec.PublicID, + APIMode: spec.APIMode, + DesiredReleaseID: spec.DesiredReleaseID, + DesiredBlueprintDigest: spec.DesiredBlueprintDigest, + DesiredInstances: spec.DesiredInstances, + MinServing: spec.MinServing, + MaxSurge: spec.MaxSurge, + MaxRepair: spec.MaxRepair, + } + return cs.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(&pool).Error +} + +func (s TrinoPoolSpec) validate() error { + if s.PoolID == "" || s.PublicID == "" { + return errors.New("trino pool spec requires a pool identity") + } + if s.APIMode != TrinoPoolAPIModeLegacy && s.APIMode != TrinoPoolAPIModeShared { + return fmt.Errorf("unsupported trino pool api mode %q", s.APIMode) + } + // A desired count of zero is missing configuration, never an instruction to + // empty the pool. Callers that cannot resolve a count must freeze instead. + if s.DesiredInstances < 1 { + return errors.New("trino pool spec requires a positive desired instance count") + } + if s.MinServing < 1 || s.MinServing > s.DesiredInstances { + return errors.New("trino pool spec requires a minimum serving count between one and the desired count") + } + if s.MaxSurge < 0 || s.MaxRepair < 0 { + return errors.New("trino pool spec requires non-negative budgets") + } + return nil +} + +// GetTrinoPool returns the pool row, or nil when the pool is unknown. +func (cs *ConfigStore) GetTrinoPool(ctx context.Context, poolID string) (*TrinoPool, error) { + var pool TrinoPool + err := cs.db.WithContext(ctx).Where("pool_id = ?", poolID).First(&pool).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &pool, nil +} + +// RecordTrinoPoolPublicationRevision records the catalog revision the pool has +// PUBLISHED, which is what a candidate must have applied before it may be +// admitted. +// +// It is monotonic: a revision behind the recorded one is ignored rather than +// treated as a regression, because catalog mutations commit in the catalog +// store's own transaction and two concurrent publications can report their +// revisions out of order. Moving the gate backwards would certify a coordinator +// that is missing the newest tenant. +func (cs *ConfigStore) RecordTrinoPoolPublicationRevision(ctx context.Context, lease TrinoPoolLease, poolID string, revision int64) error { + if poolID != lease.PoolID { + return fmt.Errorf("%w: revision belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + if revision < 0 { + return errors.New("a publication revision cannot be negative") + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, pool *TrinoPool) error { + if revision <= pool.PublicationRevision { + return nil + } + return tx.Model(&TrinoPool{}).Where("pool_id = ?", poolID). + Updates(map[string]any{"publication_revision": revision, "updated_at": time.Now().UTC()}).Error + }) +} + +// FreezeTrinoPool holds the pool at its last-good state. This is what missing +// or invalid desired configuration does: no creates, no drains, no deletes, and +// explicitly NOT a desired count of zero. +func (cs *ConfigStore) FreezeTrinoPool(ctx context.Context, lease TrinoPoolLease, poolID, reason string) error { + if reason == "" { + return errors.New("freezing a trino pool requires a reason") + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Model(&TrinoPool{}).Where("pool_id = ?", poolID). + Updates(map[string]any{"frozen": true, "frozen_reason": reason, "updated_at": time.Now().UTC()}).Error + }) +} + +// ThawTrinoPool clears the freeze once desired configuration is readable again. +func (cs *ConfigStore) ThawTrinoPool(ctx context.Context, lease TrinoPoolLease, poolID string) error { + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Model(&TrinoPool{}).Where("pool_id = ? AND frozen", poolID). + Updates(map[string]any{"frozen": false, "frozen_reason": "", "updated_at": time.Now().UTC()}).Error + }) +} + +// AcquireTrinoPoolAuthority bumps the pool's authority epoch and records the new +// owner. The bump is what invalidates a previous leader's in-flight writes, so +// it happens under the pool row lock: a takeover either lands before an old +// write's CAS or makes that CAS fail. A lease expiry on its own decides nothing. +func (cs *ConfigStore) AcquireTrinoPoolAuthority(ctx context.Context, poolID, owner string) (TrinoPoolLease, error) { + if owner == "" { + return TrinoPoolLease{}, errors.New("acquiring trino pool authority requires an owner identity") + } + var lease TrinoPoolLease + err := cs.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + pool, err := lockTrinoPool(ctx, tx, poolID) + if err != nil { + return err + } + epoch := pool.AuthorityEpoch + 1 + if err := tx.Model(&TrinoPool{}).Where("pool_id = ?", poolID).Updates(map[string]any{ + "authority_epoch": epoch, "authority_owner": owner, "updated_at": time.Now().UTC(), + }).Error; err != nil { + return err + } + lease = TrinoPoolLease{PoolID: poolID, Owner: owner, Epoch: epoch} + return nil + }) + return lease, err +} + +// lockTrinoPool takes the pool row lock. Every fenced mutation starts here, so +// all of them serialize against each other and against a takeover. +func lockTrinoPool(ctx context.Context, tx *gorm.DB, poolID string) (*TrinoPool, error) { + var pool TrinoPool + err := tx.WithContext(ctx).Clauses(clause.Locking{Strength: "UPDATE"}). + Where("pool_id = ?", poolID).First(&pool).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("%w: pool %q is not configured", ErrTrinoPoolConflict, poolID) + } + if err != nil { + return nil, err + } + return &pool, nil +} + +// checkLease verifies the caller still holds authority. It is called inside the +// pool row lock, so the answer cannot change under the caller's feet. +func checkLease(pool *TrinoPool, lease TrinoPoolLease) error { + if pool.PoolID != lease.PoolID || pool.AuthorityEpoch != lease.Epoch || pool.AuthorityOwner != lease.Owner { + return fmt.Errorf("%w: authority is %q at epoch %d, caller holds %q at epoch %d", + ErrTrinoPoolConflict, pool.AuthorityOwner, pool.AuthorityEpoch, lease.Owner, lease.Epoch) + } + return nil +} + +// withPoolAuthority runs fn under the pool row lock with the lease verified. +func (cs *ConfigStore) withPoolAuthority(ctx context.Context, lease TrinoPoolLease, fn func(*gorm.DB, *TrinoPool) error) error { + return cs.withPoolAuthorityTx(ctx, lease, "", fn) +} + +// withPoolAuthorityTx is withPoolAuthority at an explicit isolation level. +// +// Most lifecycle writes need only the row lock, so they run at the default. +// Building a projection needs more: it reads several tables that other code +// paths write, and under READ COMMITTED those reads can straddle a write and +// produce a state the database never had. +func (cs *ConfigStore) withPoolAuthorityTx( + ctx context.Context, + lease TrinoPoolLease, + isolation string, + fn func(*gorm.DB, *TrinoPool) error, +) error { + return cs.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if isolation != "" { + if err := tx.Exec("SET TRANSACTION ISOLATION LEVEL " + isolation).Error; err != nil { + return fmt.Errorf("set transaction isolation: %w", err) + } + } + pool, err := lockTrinoPool(ctx, tx, lease.PoolID) + if err != nil { + return err + } + if err := checkLease(pool, lease); err != nil { + return err + } + return fn(tx, pool) + }) +} + +// withSerializedRetry retries a transaction that the database refused for +// concurrency reasons. +// +// A snapshot transaction can be aborted by a concurrent writer, which is the +// database doing its job rather than a fault: the work is simply re-read and +// re-done. The bound is small and the error is surfaced afterwards, so a +// genuinely contended or broken path fails visibly instead of spinning. +func (cs *ConfigStore) withSerializedRetry(ctx context.Context, fn func(attempt int) error) error { + const attempts = 3 + var err error + for attempt := 1; attempt <= attempts; attempt++ { + err = fn(attempt) + if err == nil || !isSerializationFailure(err) { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + } + return err +} + +// isSerializationFailure reports PostgreSQL's class 40 - serialization failure +// and deadlock - which say "try again", not "this is wrong". +func isSerializationFailure(err error) bool { + if err == nil { + return false + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code == "40001" || pgErr.Code == "40P01" + } + return false +} diff --git a/controlplane/configstore/trino_pool_instances.go b/controlplane/configstore/trino_pool_instances.go new file mode 100644 index 000000000..33154a64e --- /dev/null +++ b/controlplane/configstore/trino_pool_instances.go @@ -0,0 +1,352 @@ +package configstore + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/posthog/duckgres/controlplane/trinopool" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// ListTrinoPoolInstances returns every instance of a pool, including terminal +// tombstones. The planner needs the tombstones to tell "three instances have +// been retired here" apart from "three instances are running". +func (cs *ConfigStore) ListTrinoPoolInstances(ctx context.Context, poolID string) ([]TrinoPoolInstance, error) { + var instances []TrinoPoolInstance + err := cs.db.WithContext(ctx).Where("pool_id = ?", poolID).Order("created_at, instance_id").Find(&instances).Error + return instances, err +} + +// GetTrinoPoolInstance returns one instance, or nil when it is unknown. +func (cs *ConfigStore) GetTrinoPoolInstance(ctx context.Context, instanceID string) (*TrinoPoolInstance, error) { + var instance TrinoPoolInstance + err := cs.db.WithContext(ctx).Where("instance_id = ?", instanceID).First(&instance).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &instance, nil +} + +// CreateTrinoPoolInstance persists an instance identity BEFORE anything is +// created in Kubernetes. That ordering is what makes a lost create response +// recoverable: the name is deterministic and already recorded, so the next tick +// reads the object back instead of creating a second one. +// +// The identity is never reused, including after retirement — the primary key +// covers terminal rows, and the partial unique index refuses to hand a live +// endpoint to a second instance. +func (cs *ConfigStore) CreateTrinoPoolInstance(ctx context.Context, lease TrinoPoolLease, spec TrinoPoolInstanceSpec) error { + if err := spec.validate(); err != nil { + return err + } + if spec.PoolID != lease.PoolID { + return fmt.Errorf("%w: instance belongs to pool %q, lease covers %q", ErrTrinoPoolConflict, spec.PoolID, lease.PoolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + snapshot := spec.BlueprintSnapshot + if snapshot == "" { + snapshot = "{}" + } + instance := TrinoPoolInstance{ + InstanceID: spec.InstanceID, + PoolID: spec.PoolID, + ReleaseID: spec.ReleaseID, + SpecDigest: spec.SpecDigest, + BlueprintSnapshot: snapshot, + Phase: string(spec.Phase), + PhaseChangedAt: time.Now().UTC(), + OwnerEpoch: lease.Epoch, + Repair: spec.Repair, + RepairFor: spec.RepairFor, + EndpointURL: spec.EndpointURL, + ValidationReceipt: "{}", + RetirementReceipt: "{}", + } + return tx.Create(&instance).Error + }) +} + +func (s TrinoPoolInstanceSpec) validate() error { + if s.InstanceID == "" || len(s.InstanceID) > 63 { + return errors.New("trino pool instance requires an identity") + } + if s.ReleaseID == "" || s.SpecDigest == "" { + return errors.New("trino pool instance requires a release and spec digest") + } + if !s.Phase.Valid() { + return fmt.Errorf("unknown trino pool instance phase %q", s.Phase) + } + return nil +} + +// AdvanceTrinoPoolInstance moves an instance between phases. The transition is +// validated against the lifecycle first (so an illegal move never reaches the +// database at all), then applied as a CAS on the CURRENT phase and the pool's +// authority epoch. +// +// The expected-phase CAS is what makes concurrent operators safe: whoever reads +// PENDING and writes CREATING first wins, and the loser is told it lost rather +// than overwriting a decision it never saw. +func (cs *ConfigStore) AdvanceTrinoPoolInstance( + ctx context.Context, + lease TrinoPoolLease, + instanceID string, + from, to trinopool.Phase, + updates map[string]any, +) error { + if err := trinopool.ValidateTransition(from, to); err != nil { + return err + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + assignments := map[string]any{ + "phase": string(to), + "phase_changed_at": time.Now().UTC(), + "owner_epoch": lease.Epoch, + "updated_at": time.Now().UTC(), + } + for key, value := range updates { + if _, reserved := assignments[key]; reserved { + return fmt.Errorf("update key %q is owned by the phase transition", key) + } + assignments[key] = value + } + result := tx.Model(&TrinoPoolInstance{}). + Where("instance_id = ? AND pool_id = ? AND phase = ?", instanceID, lease.PoolID, string(from)). + Updates(assignments) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("%w: instance %q is no longer in phase %s", ErrTrinoPoolConflict, instanceID, from) + } + return nil + }) +} + +// RecordTrinoPoolInstanceFields checkpoints observed facts that are not phase +// changes: Kubernetes UIDs after a create, the coordinator's process identity +// after a probe, a Gateway incarnation after a read-back. Still fenced, because +// a superseded leader's observations are no more trustworthy than its writes. +func (cs *ConfigStore) RecordTrinoPoolInstanceFields(ctx context.Context, lease TrinoPoolLease, instanceID string, updates map[string]any) error { + if len(updates) == 0 { + return nil + } + if _, forbidden := updates["phase"]; forbidden { + return errors.New("phase changes must go through AdvanceTrinoPoolInstance") + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + assignments := map[string]any{"updated_at": time.Now().UTC()} + for key, value := range updates { + assignments[key] = value + } + result := tx.Model(&TrinoPoolInstance{}). + Where("instance_id = ? AND pool_id = ?", instanceID, lease.PoolID). + Updates(assignments) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("%w: instance %q is unknown", ErrTrinoPoolConflict, instanceID) + } + return nil + }) +} + +// TrinoProjectionSources is one transaction's view of the projection's source +// rows. +// +// Reread repeats the read through the SAME transaction. Production does not +// need it - the projection is built once, from Orgs - but it is what lets a +// test demonstrate the property this transaction exists for: a write committed +// by somebody else in between must not become visible half way through. +type TrinoProjectionSources struct { + Orgs []TrinoEnabledOrg + Reread func() ([]TrinoEnabledOrg, error) +} + +// AcceptTrinoPoolProjectionWith builds the projection and accepts it in ONE +// transaction. +// +// The callback receives the projection's source rows read inside that +// transaction, under the pool's authority lock, and returns the digest of the +// bytes it built from exactly those rows. Allocating a revision for content +// read at any other moment would number bytes nobody can prove were current - +// the "fresh counter on a stale buffer" that makes the whole fence decorative. +// +// The caller keeps the bytes it built in the callback and writes THOSE, under +// the returned revision. +func (cs *ConfigStore) AcceptTrinoPoolProjectionWith( + ctx context.Context, + lease TrinoPoolLease, + build func(orgs []TrinoEnabledOrg) (digest string, err error), +) (int64, string, error) { + return cs.AcceptTrinoPoolProjectionFrom(ctx, lease, func(sources TrinoProjectionSources) (string, error) { + return build(sources.Orgs) + }) +} + +// AcceptTrinoPoolProjectionFrom is AcceptTrinoPoolProjectionWith with the +// transaction's source view handed to the builder. +func (cs *ConfigStore) AcceptTrinoPoolProjectionFrom( + ctx context.Context, + lease TrinoPoolLease, + build func(sources TrinoProjectionSources) (digest string, err error), +) (int64, string, error) { + if build == nil { + return 0, "", errors.New("accepting a projection requires a builder") + } + var ( + revision int64 + digest string + ) + // REPEATABLE READ, not the default. + // + // The pool row lock serializes acceptances against each other, but it does + // not lock the source of the projection: orgs, users and teams are written + // by entirely different code paths. Under READ COMMITTED the separate + // SELECTs this builds from can straddle such a write and produce a + // projection that never existed as a state of the database - and it would + // be accepted as the coherent one. A snapshot makes every read in the + // transaction see one instant. + err := cs.withSerializedRetry(ctx, func(attempt int) error { + revision, digest = 0, "" + return cs.withPoolAuthorityTx(ctx, lease, "REPEATABLE READ", func(tx *gorm.DB, _ *TrinoPool) error { + orgs, err := cs.listTrinoEnabledOrgsCoherently(tx) + if err != nil { + return err + } + digest, err = build(TrinoProjectionSources{ + Orgs: orgs, + Reread: func() ([]TrinoEnabledOrg, error) { return cs.listTrinoEnabledOrgsCoherently(tx) }, + }) + if err != nil { + return err + } + if digest == "" { + return errors.New("the projection builder produced no digest") + } + revision, err = acceptProjectionTx(tx, lease, digest) + return err + }) + }) + return revision, digest, err +} + +// AcceptTrinoPoolProjection records an already-built projection's digest and +// returns the revision it is accepted at. +// +// The revision is the ORDER the fence needs: a digest identifies a projection +// but cannot say which of two came first, and every control plane builds one +// from its own view - so without an authority assigning an order, a replica +// that is behind cannot tell that it is. An unchanged digest keeps its +// revision, so a steady-state tick allocates nothing. +func (cs *ConfigStore) AcceptTrinoPoolProjection(ctx context.Context, lease TrinoPoolLease, digest string) (int64, error) { + if digest == "" { + return 0, errors.New("a projection digest is required") + } + var revision int64 + err := cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + var err error + revision, err = acceptProjectionTx(tx, lease, digest) + return err + }) + return revision, err +} + +// acceptProjectionTx records the accepted projection inside an already-fenced +// transaction and returns the revision it is accepted at. +func acceptProjectionTx(tx *gorm.DB, lease TrinoPoolLease, digest string) (int64, error) { + var revision int64 + existing := TrinoPoolProjection{} + err := tx.Where("pool_id = ?", lease.PoolID).First(&existing).Error + switch { + case err == nil: + if existing.AcceptedDigest == digest { + // Already the accepted projection. Allocating a new revision for + // identical content would make every tick look like a change and + // every replica refuse to serve for a moment. + return existing.AcceptedRevision, nil + } + revision = existing.AcceptedRevision + 1 + case errors.Is(err, gorm.ErrRecordNotFound): + revision = 1 + default: + return 0, err + } + if err := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "pool_id"}}, + DoUpdates: clause.Assignments(map[string]any{ + "authority_epoch": lease.Epoch, + "accepted_revision": revision, + "accepted_digest": digest, + "updated_at": time.Now().UTC(), + }), + }).Create(&TrinoPoolProjection{ + PoolID: lease.PoolID, + AuthorityEpoch: lease.Epoch, + AcceptedRevision: revision, + AcceptedDigest: digest, + UpdatedAt: time.Now().UTC(), + }).Error; err != nil { + return 0, err + } + return revision, nil +} + +// AdvanceTrinoPoolProjection moves the accepted authorization-projection +// watermark forward. It is monotonic and fenced: a stale leader cannot move it, +// and nobody can move it backwards. Serving replicas compare their own snapshot +// against this value before emitting an OPA bundle, so a regressing body is +// never produced in the first place. +func (cs *ConfigStore) AdvanceTrinoPoolProjection(ctx context.Context, lease TrinoPoolLease, revision int64, digest string) error { + if revision < 1 { + return errors.New("projection revision must be positive") + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + projection := TrinoPoolProjection{ + PoolID: lease.PoolID, + AuthorityEpoch: lease.Epoch, + AcceptedRevision: revision, + AcceptedDigest: digest, + UpdatedAt: time.Now().UTC(), + } + result := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "pool_id"}}, + DoUpdates: clause.Assignments(map[string]any{ + "authority_epoch": lease.Epoch, + "accepted_revision": revision, + "accepted_digest": digest, + "updated_at": time.Now().UTC(), + }), + // Monotonic: an older revision is not an update, it is a no-op that + // the caller is told about. + Where: clause.Where{Exprs: []clause.Expression{ + gorm.Expr("duckgres_trino_pool_projection.accepted_revision < ?", revision), + }}, + }).Create(&projection) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("%w: projection revision %d does not advance the accepted watermark", ErrTrinoPoolConflict, revision) + } + return nil + }) +} + +// GetTrinoPoolProjection returns the accepted watermark. A zero value means no +// projection has been accepted yet. +func (cs *ConfigStore) GetTrinoPoolProjection(ctx context.Context, poolID string) (TrinoPoolProjection, error) { + var projection TrinoPoolProjection + err := cs.db.WithContext(ctx).Where("pool_id = ?", poolID).First(&projection).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return TrinoPoolProjection{PoolID: poolID}, nil + } + return projection, err +} diff --git a/controlplane/configstore/trino_pool_models.go b/controlplane/configstore/trino_pool_models.go new file mode 100644 index 000000000..37bd92ea2 --- /dev/null +++ b/controlplane/configstore/trino_pool_models.go @@ -0,0 +1,333 @@ +package configstore + +import ( + "errors" + "time" + + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// Durable state of the shared Trino compute pool. Migration 000041 owns the +// physical schema. Everything here is inert unless a registered cell declares +// shared-pool mode and the feature flag is on. + +// ErrTrinoPoolConflict is returned when a fenced write loses: a superseded +// authority epoch, a failed phase CAS, or a watermark that would regress. It is +// never a reason to retry with a fresher epoch read from the database — the +// caller has lost leadership and must stop writing. +var ErrTrinoPoolConflict = errors.New("trino pool conflict") + +// ErrTrinoPoolIntentChanged is returned when an operation or step id is reused +// with different content. That is a caller bug, not a replay, and applying it +// would perform an effect nobody recorded an intent for. +var ErrTrinoPoolIntentChanged = errors.New("trino pool operation was replayed with different content") + +// ErrTrinoPoolStaleGeneration is returned when a desired specification would +// move the recorded generation BACKWARDS. +// +// It is deliberately NOT an ErrTrinoPoolConflict: losing a fence means this +// process is no longer the authority and must stop, whereas this means the +// authority is holding a specification the store considers older. Conflating +// the two ended the leadership term on every tick over what is a configuration +// problem, and handed the pool to a replica that would do exactly the same. +var ErrTrinoPoolStaleGeneration = errors.New("trino pool desired generation is behind the published generation") + +// API modes. `legacy` keeps today's fixed blue/green behavior for the cell. +const ( + TrinoPoolAPIModeLegacy = "legacy" + TrinoPoolAPIModeShared = "shared-pool" +) + +// Operation kinds. +const ( + TrinoPoolOperationReplace = "replace" + TrinoPoolOperationScaleUp = "scale_up" + TrinoPoolOperationRepair = "repair" + TrinoPoolOperationRetire = "retire" + TrinoPoolOperationPublish = "publish" +) + +// Publication states. Desired, published and admitted are distinct facts; a +// single boolean cannot express "enabled but not yet allowed to query". +const ( + TrinoPublicationPending = "pending" + TrinoPublicationPublished = "published" + TrinoPublicationAdmitting = "admitting" + TrinoPublicationAdmitted = "admitted" + TrinoPublicationFailed = "failed" + // TrinoPublicationRevoked is a tenant whose admission was withdrawn. The + // row is KEPT: a deleted row would read as "never published" and the next + // tick would republish the binding of a tenant that is meant to be gone. + TrinoPublicationRevoked = "revoked" +) + +// TrinoPool is the desired specification plus the operator's runtime state. +type TrinoPool struct { + PoolID string `gorm:"primaryKey;column:pool_id"` + PublicID string `gorm:"column:public_id"` + APIMode string `gorm:"column:api_mode"` + DesiredReleaseID string `gorm:"column:desired_release_id"` + DesiredBlueprintDigest string `gorm:"column:desired_blueprint_digest"` + DesiredInstances int `gorm:"column:desired_instances"` + MinServing int `gorm:"column:min_serving"` + MaxSurge int `gorm:"column:max_surge"` + MaxRepair int `gorm:"column:max_repair"` + // DesiredGeneration orders desired-state CONTENT independently of who + // wrote it, so a leader carrying an older configuration cannot publish it + // over a newer one just because it legitimately holds the fence. + DesiredGeneration int64 `gorm:"column:desired_generation"` + AuthorityEpoch int64 `gorm:"column:authority_epoch"` + AuthorityOwner string `gorm:"column:authority_owner"` + PublicationRevision int64 `gorm:"column:publication_revision"` + AdmittedRevision int64 `gorm:"column:admitted_revision"` + Frozen bool `gorm:"column:frozen"` + FrozenReason string `gorm:"column:frozen_reason"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (TrinoPool) TableName() string { return "duckgres_trino_pools" } + +// TrinoPoolSpec is the desired configuration as resolved from the registry and +// the blueprint. It deliberately carries no runtime state: applying it must not +// disturb the authority epoch, the freeze flag, or anything an operator owns. +type TrinoPoolSpec struct { + // Generation is the ordering of this desired content. It must increase + // whenever the spec changes; a publication that does not advance it is + // refused as stale. + Generation int64 + PoolID string + PublicID string + APIMode string + DesiredReleaseID string + DesiredBlueprintDigest string + DesiredInstances int + MinServing int + MaxSurge int + MaxRepair int +} + +// TrinoPoolLease is proof of current authority over a pool. Every fenced write +// takes one; a write whose epoch no longer matches the stored epoch is refused. +type TrinoPoolLease struct { + PoolID string + Owner string + Epoch int64 +} + +// TrinoPoolInstance is one immutable compute instance. +type TrinoPoolInstance struct { + InstanceID string `gorm:"primaryKey;column:instance_id"` + PoolID string `gorm:"column:pool_id"` + ReleaseID string `gorm:"column:release_id"` + SpecDigest string `gorm:"column:spec_digest"` + BlueprintSnapshot string `gorm:"column:blueprint_snapshot;type:jsonb"` + Phase string `gorm:"column:phase"` + PhaseChangedAt time.Time `gorm:"column:phase_changed_at"` + OwnerEpoch int64 `gorm:"column:owner_epoch"` + Repair bool `gorm:"column:repair"` + // RepairFor names the failed instance this one replaces. The Gateway + // charges an activation to the repair budget only when it is set; without + // it a repair spends the single planned surge instead. + RepairFor string `gorm:"column:repair_for"` + FailureReason string `gorm:"column:failure_reason"` + CoordinatorDeploymentName string `gorm:"column:coordinator_deployment_name"` + CoordinatorDeploymentUID string `gorm:"column:coordinator_deployment_uid"` + WorkerDeploymentName string `gorm:"column:worker_deployment_name"` + WorkerDeploymentUID string `gorm:"column:worker_deployment_uid"` + ServiceName string `gorm:"column:service_name"` + ServiceUID string `gorm:"column:service_uid"` + ConfigMapName string `gorm:"column:config_map_name"` + ConfigMapUID string `gorm:"column:config_map_uid"` + WorkerConfigMapName string `gorm:"column:worker_config_map_name"` + WorkerConfigMapUID string `gorm:"column:worker_config_map_uid"` + CoordinatorPodUID string `gorm:"column:coordinator_pod_uid"` + CoordinatorNodeID string `gorm:"column:coordinator_node_id"` + // CoordinatorID is the coordinator identity the GATEWAY observed when the + // member registered. It is a distinct value from the node id, and a loss + // claim has to carry both exactly as the Gateway recorded them, or the + // evidence is refused and the member keeps its live slot forever. + CoordinatorID string `gorm:"column:coordinator_id"` + // CoordinatorContainerID is the container instance that hosted the admitted + // process. Termination records name a container instance, so this is what + // ties one to the process the Gateway admitted rather than to some earlier + // restart of the same pod. + CoordinatorContainerID string `gorm:"column:coordinator_container_id"` + CoordinatorBootID string `gorm:"column:coordinator_boot_id"` + EndpointURL string `gorm:"column:endpoint_url"` + // TLSServerName is retained on the row for the fixed-cell path only. A + // pooled instance is reached over plain in-cluster HTTP and has no + // certificate of its own, so the pool never sets it. + TLSServerName string `gorm:"column:tls_server_name"` + GatewayIncarnation string `gorm:"column:gateway_incarnation"` + GatewayBackendName string `gorm:"column:gateway_backend_name"` + GatewayState string `gorm:"column:gateway_state"` + GatewayGeneration int64 `gorm:"column:gateway_generation"` + AppliedCatalogRevision int64 `gorm:"column:applied_catalog_revision"` + ValidationReceipt string `gorm:"column:validation_receipt;type:jsonb"` + ValidatedAt *time.Time `gorm:"column:validated_at"` + RetirementReceipt string `gorm:"column:retirement_receipt;type:jsonb"` + LastError string `gorm:"column:last_error"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (TrinoPoolInstance) TableName() string { return "duckgres_trino_pool_instances" } + +// View projects the row onto the planner's read-only input. +func (i TrinoPoolInstance) View() trinopool.InstanceView { + return trinopool.InstanceView{ + ID: i.InstanceID, + Phase: trinopool.Phase(i.Phase), + ReleaseID: i.ReleaseID, + Repair: i.Repair, + CreatedAt: i.CreatedAt.UnixNano(), + } +} + +// TrinoPoolInstanceSpec is the immutable identity of a new instance. +type TrinoPoolInstanceSpec struct { + InstanceID string + PoolID string + ReleaseID string + SpecDigest string + BlueprintSnapshot string + Phase trinopool.Phase + Repair bool + RepairFor string + EndpointURL string +} + +// TrinoPoolOperation is a durable reconcile intent that outlives the leader. +type TrinoPoolOperation struct { + OperationID string `gorm:"primaryKey;column:operation_id"` + PoolID string `gorm:"column:pool_id"` + InstanceID string `gorm:"column:instance_id"` + Kind string `gorm:"column:kind"` + IntentHash string `gorm:"column:intent_hash"` + OwnerEpoch int64 `gorm:"column:owner_epoch"` + Step string `gorm:"column:step"` + Phase string `gorm:"column:phase"` + Receipts string `gorm:"column:receipts;type:jsonb"` + LastError string `gorm:"column:last_error"` + Attempts int64 `gorm:"column:attempts"` + NextAttemptAt *time.Time `gorm:"column:next_attempt_at"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` + TerminalAt *time.Time `gorm:"column:terminal_at"` + + // Replayed reports that this call found an existing identical operation + // rather than creating one. It is not a column. + Replayed bool `gorm:"-"` +} + +func (TrinoPoolOperation) TableName() string { return "duckgres_trino_pool_operations" } + +// TrinoPoolOperationSpec is the immutable intent of an operation. +type TrinoPoolOperationSpec struct { + OperationID string + PoolID string + InstanceID string + Kind string + IntentHash string +} + +// TrinoPoolOperationStep is one idempotent step inside an operation. Scoping +// replay identity to the step is what lets a resumed operation re-run only the +// step that was interrupted. +type TrinoPoolOperationStep struct { + OperationID string `gorm:"primaryKey;column:operation_id"` + StepID string `gorm:"primaryKey;column:step_id"` + PayloadHash string `gorm:"column:payload_hash"` + Outcome string `gorm:"column:outcome"` + Result string `gorm:"column:result;type:jsonb"` + RecordedAt time.Time `gorm:"column:recorded_at"` + + Replayed bool `gorm:"-"` +} + +func (TrinoPoolOperationStep) TableName() string { return "duckgres_trino_pool_operation_steps" } + +// Recorded step outcomes. They are defined here, next to the row they are +// written into, because the store itself has to distinguish them: an UNKNOWN +// step may be completed by a later attempt, a decided one never is. +const ( + // TrinoPoolStepOutcomeUnknown is recorded BEFORE the external effect. It + // means "this may or may not have happened", which is the only honest + // answer to a lost response. + TrinoPoolStepOutcomeUnknown = "UNKNOWN" + // TrinoPoolStepOutcomeOK is a completed effect, and its result is the + // answer a later attempt reads back instead of repeating the call. + TrinoPoolStepOutcomeOK = "OK" + // TrinoPoolStepOutcomeFailed is a REFUSAL. Retrying cannot change it. + TrinoPoolStepOutcomeFailed = "FAILED" +) + +// TrinoPoolPublication is one warehouse's publication state on a pool. +type TrinoPoolPublication struct { + PoolID string `gorm:"primaryKey;column:pool_id"` + OrgID string `gorm:"primaryKey;column:org_id"` + DesiredRevision int64 `gorm:"column:desired_revision"` + PublishedRevision int64 `gorm:"column:published_revision"` + AdmittedRevision int64 `gorm:"column:admitted_revision"` + PublicationID string `gorm:"column:publication_id"` + PublicationOperationID string `gorm:"column:publication_operation_id"` + // PrincipalRevision is the binding (the tenant's principal set) last + // published to the Gateway, and TargetRevision the configuration revision + // the open barrier requires every serving member to acknowledge. + // AdmittedTargetRevision is the last one that actually committed. + // + // They are durable rather than remembered in the leader's memory: a + // restart or a leadership move would otherwise either republish blindly or + // assume an admission that never happened. + PrincipalRevision string `gorm:"column:principal_revision"` + TargetRevision string `gorm:"column:target_revision"` + AdmittedTargetRevision string `gorm:"column:admitted_target_revision"` + // Attempt is a monotone occurrence counter. It is part of every durable + // step identity this tenant's barrier and revocations use, so a reopened + // barrier - or a second revocation after the tenant was re-enabled - is a + // new operation rather than a replay that returns the first one's outcome. + Attempt int64 `gorm:"column:attempt"` + // PendingIntent names the request the open occurrence stands for while its + // outcome is unknown: "principals", "revoke", or empty for none. + // + // A lost response is not a finished request. Until the Gateway gives a + // definite answer, the next pass reissues THAT occurrence's step identity + // rather than minting a new one, so a request still executing at the + // Gateway cannot commit after a newer desired intent has already been + // checkpointed here. + PendingIntent string `gorm:"column:pending_intent"` + // PendingPayload is that request's body, so the reissue is byte-identical + // to the original: the same step identity carrying the same bytes is an + // ordinary replay, which is far easier to reason about than sending a new + // body under an old identity and reading the refusal as a success. + // + // It carries principal identifiers and the revision naming them - never a + // password or a hash. + PendingPayload string `gorm:"column:pending_payload;type:jsonb"` + // Attempts and NextAttemptAt are this tenant's own durable backoff. The + // driver takes one tenant at a time, so without them a permanently failing + // warehouse is retried every tick and starves every tenant behind it. + Attempts int64 `gorm:"column:attempts"` + NextAttemptAt *time.Time `gorm:"column:next_attempt_at"` + State string `gorm:"column:state"` + GatewayReceipt string `gorm:"column:gateway_receipt;type:jsonb"` + LastError string `gorm:"column:last_error"` + CreatedAt time.Time `gorm:"column:created_at"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (TrinoPoolPublication) TableName() string { return "duckgres_trino_pool_publications" } + +// TrinoPoolProjection is the accepted authorization-projection watermark. A +// replica consults it before serving an OPA bundle and refuses to serve +// anything older, so a stale replica never produces a regressing body. +type TrinoPoolProjection struct { + PoolID string `gorm:"primaryKey;column:pool_id"` + AuthorityEpoch int64 `gorm:"column:authority_epoch"` + AcceptedRevision int64 `gorm:"column:accepted_revision"` + AcceptedDigest string `gorm:"column:accepted_digest"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (TrinoPoolProjection) TableName() string { return "duckgres_trino_pool_projection" } diff --git a/controlplane/configstore/trino_pool_operations.go b/controlplane/configstore/trino_pool_operations.go new file mode 100644 index 000000000..be8a3cf90 --- /dev/null +++ b/controlplane/configstore/trino_pool_operations.go @@ -0,0 +1,595 @@ +package configstore + +import ( + "context" + "errors" + "fmt" + "time" + + "gorm.io/gorm" +) + +// Durable operations and their steps. +// +// The whole point of these rows is that an external effect is preceded by a +// recorded intent. When a response is lost, the operator does not guess and it +// does not mint a new identity: it reads the same operation back and continues. + +// BeginTrinoPoolOperation records an intent, or returns the existing one when +// the identical intent was already recorded. A reused id with a different +// intent hash is a conflict — applying it would perform an effect nobody +// recorded, and silently overwriting it would erase the original. +func (cs *ConfigStore) BeginTrinoPoolOperation(ctx context.Context, lease TrinoPoolLease, spec TrinoPoolOperationSpec) (TrinoPoolOperation, error) { + if spec.OperationID == "" || spec.IntentHash == "" { + return TrinoPoolOperation{}, errors.New("trino pool operation requires an id and intent hash") + } + if spec.PoolID != lease.PoolID { + return TrinoPoolOperation{}, fmt.Errorf("%w: operation belongs to pool %q", ErrTrinoPoolConflict, spec.PoolID) + } + switch spec.Kind { + case TrinoPoolOperationReplace, TrinoPoolOperationScaleUp, TrinoPoolOperationRepair, + TrinoPoolOperationRetire, TrinoPoolOperationPublish: + default: + return TrinoPoolOperation{}, fmt.Errorf("unsupported trino pool operation kind %q", spec.Kind) + } + + var operation TrinoPoolOperation + err := cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + existing := TrinoPoolOperation{} + err := tx.Where("operation_id = ?", spec.OperationID).First(&existing).Error + switch { + case err == nil: + if existing.IntentHash != spec.IntentHash { + return fmt.Errorf("%w: operation %q", ErrTrinoPoolIntentChanged, spec.OperationID) + } + existing.Replayed = true + operation = existing + return nil + case !errors.Is(err, gorm.ErrRecordNotFound): + return err + } + + created := TrinoPoolOperation{ + OperationID: spec.OperationID, + PoolID: spec.PoolID, + InstanceID: spec.InstanceID, + Kind: spec.Kind, + IntentHash: spec.IntentHash, + OwnerEpoch: lease.Epoch, + Phase: "pending", + Receipts: "{}", + } + if err := tx.Create(&created).Error; err != nil { + return err + } + operation = created + return nil + }) + return operation, err +} + +// GetTrinoPoolOperation reads an operation back. This is the answer to a lost +// response: the recorded outcome, not a fresh attempt. +func (cs *ConfigStore) GetTrinoPoolOperation(ctx context.Context, operationID string) (*TrinoPoolOperation, error) { + var operation TrinoPoolOperation + err := cs.db.WithContext(ctx).Where("operation_id = ?", operationID).First(&operation).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &operation, nil +} + +// ListOpenTrinoPoolOperations returns the operations a new leader must resume. +func (cs *ConfigStore) ListOpenTrinoPoolOperations(ctx context.Context, poolID string) ([]TrinoPoolOperation, error) { + var operations []TrinoPoolOperation + err := cs.db.WithContext(ctx). + Where("pool_id = ? AND terminal_at IS NULL", poolID). + Order("created_at, operation_id").Find(&operations).Error + return operations, err +} + +// RecordTrinoPoolOperationStep records one step's outcome, idempotently. A +// replay with the identical payload returns the recorded result — that is how a +// lost Gateway response becomes a lookup rather than a second mutation. A +// different payload under the same step id is a conflict. +func (cs *ConfigStore) RecordTrinoPoolOperationStep(ctx context.Context, lease TrinoPoolLease, operationID, stepID, payloadHash, outcome, result string) (TrinoPoolOperationStep, error) { + if operationID == "" || stepID == "" || payloadHash == "" { + return TrinoPoolOperationStep{}, errors.New("trino pool step requires an operation, step id and payload hash") + } + if result == "" { + result = "{}" + } + var step TrinoPoolOperationStep + // Fenced like every other lifecycle write: a superseded leader recording + // step outcomes into a live operation would make the read-back path report + // its abandoned attempt as the operation's result. + err := cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + existing := TrinoPoolOperationStep{} + err := tx.Where("operation_id = ? AND step_id = ?", operationID, stepID).First(&existing).Error + switch { + case err == nil: + if existing.PayloadHash != payloadHash { + return fmt.Errorf("%w: operation %q step %q", ErrTrinoPoolIntentChanged, operationID, stepID) + } + // A step is recorded UNKNOWN before the effect and re-recorded with + // the outcome after it. Returning the stored row unchanged made that + // second call a no-op, so a step could never leave UNKNOWN and the + // cross-leader read-back that keys on OK was unreachable: every + // retry re-called the external system and relied on ITS replay + // guard instead of this journal. + // + // An outcome only ever advances out of UNKNOWN. A terminal outcome + // is never overwritten - re-deciding a recorded OK or FAILED is + // exactly the rewriting of history the journal exists to prevent. + if existing.Outcome == TrinoPoolStepOutcomeUnknown && outcome != "" && outcome != TrinoPoolStepOutcomeUnknown { + if err := tx.Model(&TrinoPoolOperationStep{}). + Where("operation_id = ? AND step_id = ?", operationID, stepID). + Updates(map[string]any{ + "outcome": outcome, + "result": result, + "recorded_at": time.Now().UTC(), + }).Error; err != nil { + return err + } + existing.Outcome = outcome + existing.Result = result + } + existing.Replayed = true + step = existing + return nil + case !errors.Is(err, gorm.ErrRecordNotFound): + return err + } + created := TrinoPoolOperationStep{ + OperationID: operationID, + StepID: stepID, + PayloadHash: payloadHash, + Outcome: outcome, + Result: result, + RecordedAt: time.Now().UTC(), + } + if err := tx.Create(&created).Error; err != nil { + return err + } + step = created + return nil + }) + return step, err +} + +// UpdateTrinoPoolOperation checkpoints progress, an error, or the next attempt +// time. Attempts and next_attempt_at are persisted rather than kept in the +// leader's memory so a restart does not reset a backoff to zero. +func (cs *ConfigStore) UpdateTrinoPoolOperation(ctx context.Context, lease TrinoPoolLease, operationID string, updates map[string]any) error { + if len(updates) == 0 { + return nil + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + assignments := map[string]any{"updated_at": time.Now().UTC()} + for key, value := range updates { + assignments[key] = value + } + result := tx.Model(&TrinoPoolOperation{}). + Where("operation_id = ? AND pool_id = ? AND terminal_at IS NULL", operationID, lease.PoolID). + Updates(assignments) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("%w: operation %q is unknown or already terminal", ErrTrinoPoolConflict, operationID) + } + return nil + }) +} + +// FinishTrinoPoolOperation marks an operation terminal. A timeout is NOT a +// terminal success: callers pass the outcome they actually observed. +func (cs *ConfigStore) FinishTrinoPoolOperation(ctx context.Context, lease TrinoPoolLease, operationID, phase, lastError string) error { + return cs.UpdateTrinoPoolOperation(ctx, lease, operationID, map[string]any{ + "phase": phase, + "last_error": lastError, + "terminal_at": time.Now().UTC(), + }) +} + +// SetTrinoPoolPublicationDesired records that a warehouse SHOULD be published at +// a revision. Desired is not admitted: the tenant cannot query until the +// Gateway's publication barrier commits and that outcome is checkpointed here. +func (cs *ConfigStore) SetTrinoPoolPublicationDesired(ctx context.Context, lease TrinoPoolLease, poolID, orgID string, revision int64) error { + if orgID == "" { + return errors.New("publication requires an org") + } + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Exec(` + INSERT INTO duckgres_trino_pool_publications (pool_id, org_id, desired_revision, state, gateway_receipt) + VALUES (?, ?, ?, ?, '{}') + ON CONFLICT (pool_id, org_id) DO UPDATE SET + desired_revision = GREATEST(duckgres_trino_pool_publications.desired_revision, EXCLUDED.desired_revision), + updated_at = now()`, + poolID, orgID, revision, TrinoPublicationPending).Error + }) +} + +// RecordTrinoPoolPublicationAdmitted checkpoints a COMMITTED Gateway barrier. +// The Gateway receipt is authoritative from the moment it commits, so this is a +// checkpoint of something already true — never a retraction point. Recovery +// after an interrupted checkpoint re-reads the Gateway and completes; it does +// not close a gate the Gateway has opened. +func (cs *ConfigStore) RecordTrinoPoolPublicationAdmitted(ctx context.Context, lease TrinoPoolLease, poolID, orgID string, revision int64, publicationID, receipt string) error { + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + if receipt == "" { + receipt = "{}" + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + result := tx.Model(&TrinoPoolPublication{}). + Where("pool_id = ? AND org_id = ?", poolID, orgID). + Updates(map[string]any{ + "admitted_revision": revision, + "published_revision": revision, + "publication_id": publicationID, + "state": TrinoPublicationAdmitted, + "gateway_receipt": receipt, + "last_error": "", + "updated_at": time.Now().UTC(), + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("%w: publication for org %q is unknown", ErrTrinoPoolConflict, orgID) + } + return nil + }) +} + +// GetTrinoPoolPublication returns one warehouse's publication state, or nil. +func (cs *ConfigStore) GetTrinoPoolPublication(ctx context.Context, poolID, orgID string) (*TrinoPoolPublication, error) { + var publication TrinoPoolPublication + err := cs.db.WithContext(ctx).Where("pool_id = ? AND org_id = ?", poolID, orgID).First(&publication).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &publication, nil +} + +// RecordTrinoPoolTenantPrincipals checkpoints the binding a tenant's principals +// were last published under. +// +// It is durable because "already published" cannot live in a leader's memory: a +// restart or a leadership move would either republish every tenant blindly or, +// worse, treat an unpublished tenant as done. +func (cs *ConfigStore) RecordTrinoPoolTenantPrincipals(ctx context.Context, lease TrinoPoolLease, poolID, orgID, principalRevision string) error { + if orgID == "" || principalRevision == "" { + return errors.New("a principal publication requires an org and a revision") + } + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Exec(` + INSERT INTO duckgres_trino_pool_publications + (pool_id, org_id, principal_revision, state, gateway_receipt) + VALUES (?, ?, ?, ?, '{}') + ON CONFLICT (pool_id, org_id) DO UPDATE SET + principal_revision = EXCLUDED.principal_revision, + -- A revoked tenant that is published again is live again; any + -- other state stays where it was, because publishing a binding + -- is not an admission. + state = CASE WHEN duckgres_trino_pool_publications.state = ? + THEN ? ELSE duckgres_trino_pool_publications.state END, + -- The Gateway answered, so this occurrence is spent: the next + -- desired change takes a new one. + pending_intent = '', + pending_payload = '{}', + last_error = '', + updated_at = now()`, + poolID, orgID, principalRevision, TrinoPublicationPublished, + TrinoPublicationRevoked, TrinoPublicationPublished).Error + }) +} + +// RecordTrinoPoolPublicationOpen checkpoints an OPEN barrier. +// +// The identity is recorded BEFORE the Gateway call that creates it, so a lost +// response is resolved by reading that publication back rather than by opening +// a second barrier for the same tenant - which the Gateway refuses anyway, and +// which would leave the first one open forever. +// +// A LIVE barrier is a non-empty publication_id, and that is the only thing +// this writes about liveness. `state` describes the tenant's ADMISSION, which a new +// barrier does not retract: a tenant that is admitted today stays admitted +// while the barrier for its newest login runs, so an operator surface keyed on +// the state cannot flap a serving warehouse back to Provisioning for a change +// that has not landed yet. A tenant that has never been admitted moves to +// `admitting`, which is the honest answer for it. +func (cs *ConfigStore) RecordTrinoPoolPublicationOpen(ctx context.Context, lease TrinoPoolLease, poolID, orgID, publicationID, targetRevision string) error { + if orgID == "" || publicationID == "" || targetRevision == "" { + return errors.New("an open publication requires an org, a publication id and a target revision") + } + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Exec(` + INSERT INTO duckgres_trino_pool_publications + (pool_id, org_id, publication_id, target_revision, state, gateway_receipt) + VALUES (?, ?, ?, ?, ?, '{}') + ON CONFLICT (pool_id, org_id) DO UPDATE SET + publication_id = EXCLUDED.publication_id, + target_revision = EXCLUDED.target_revision, + state = CASE WHEN duckgres_trino_pool_publications.admitted_target_revision = '' + THEN ? ELSE duckgres_trino_pool_publications.state END, + updated_at = now()`, + poolID, orgID, publicationID, targetRevision, TrinoPublicationAdmitting, + TrinoPublicationAdmitting).Error + }) +} + +// ClearTrinoPoolPublicationBarrier forgets a barrier that is no longer live, +// so the driver stops treating it as the one attempt in flight. +// +// It is the durable half of abandoning an attempt. Without it the row keeps +// naming a publication the Gateway has already ABANDONED, and every later pass +// selects that same finished attempt again - which is how an open barrier that +// was blocking a member's admission stayed selected, was re-abandoned on every +// attempt, and never let the driver reach the barrier that was actually in the +// way. +// +// The tenant's ADMISSION is not touched. A tenant that was admitted stays +// admitted (its `admitted_target_revision` is the proof); one that never was +// falls back to whether its binding has been published at all. +func (cs *ConfigStore) ClearTrinoPoolPublicationBarrier(ctx context.Context, lease TrinoPoolLease, poolID, orgID string) error { + if orgID == "" { + return errors.New("clearing a barrier requires an org") + } + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Exec(` + UPDATE duckgres_trino_pool_publications SET + publication_id = '', + target_revision = '', + state = CASE + WHEN admitted_target_revision <> '' THEN state + WHEN principal_revision <> '' THEN ? + ELSE ? END, + updated_at = now() + WHERE pool_id = ? AND org_id = ?`, + TrinoPublicationPublished, TrinoPublicationPending, poolID, orgID).Error + }) +} + +// BeginTrinoPoolPublicationAttempt bumps a tenant's occurrence counter and +// returns the new value. +// +// Every durable step identity for that tenant carries it, so an attempt that +// follows an abandoned barrier - or a second revocation after the tenant was +// re-enabled - is a NEW operation. Reusing the identity would replay the first +// attempt's recorded outcome and leave the current intent unapplied, which for +// a revocation means a tenant nobody revoked stays admitted. +func (cs *ConfigStore) BeginTrinoPoolPublicationAttempt(ctx context.Context, lease TrinoPoolLease, poolID, orgID string) (int64, error) { + if orgID == "" { + return 0, errors.New("a publication attempt requires an org") + } + if poolID != lease.PoolID { + return 0, fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + var attempt int64 + err := cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Raw(` + INSERT INTO duckgres_trino_pool_publications (pool_id, org_id, attempt, state, gateway_receipt) + VALUES (?, ?, 1, ?, '{}') + ON CONFLICT (pool_id, org_id) DO UPDATE SET + attempt = duckgres_trino_pool_publications.attempt + 1, + updated_at = now() + RETURNING attempt`, + poolID, orgID, TrinoPublicationPending).Scan(&attempt).Error + }) + return attempt, err +} + +// Pending intents. A tenant has at most one request in flight, and the kind of +// request its open occurrence stands for is durable. +const ( + TrinoPublicationIntentPrincipals = "principals" + TrinoPublicationIntentRevoke = "revoke" +) + +// BeginTrinoPoolPublicationIntent opens a NEW occurrence for a request whose +// outcome will be unknown until the Gateway answers, and stores the request +// itself. +// +// It is the one write that bumps the occurrence, records what that occurrence +// stands for AND keeps its exact body, so a leader that dies between them can +// never leave an occurrence nobody can attribute or reissue. Until the intent is +// resolved the driver replays THIS occurrence - same identity, same bytes - +// instead of minting another, which is what stops a request still executing at +// the Gateway from committing after a newer intent has been checkpointed here. +// +// The payload is principal identifiers and the revision naming them. No +// credential of any kind belongs in it. +func (cs *ConfigStore) BeginTrinoPoolPublicationIntent(ctx context.Context, lease TrinoPoolLease, poolID, orgID, kind, payload string) (int64, error) { + if orgID == "" { + return 0, errors.New("a publication intent requires an org") + } + switch kind { + case TrinoPublicationIntentPrincipals, TrinoPublicationIntentRevoke: + default: + return 0, fmt.Errorf("unsupported publication intent %q", kind) + } + if payload == "" { + // An occurrence with no body could not be replayed, which is the whole + // point of recording it. + return 0, errors.New("a publication intent requires its request body") + } + if poolID != lease.PoolID { + return 0, fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + var attempt int64 + err := cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Raw(` + INSERT INTO duckgres_trino_pool_publications + (pool_id, org_id, attempt, pending_intent, pending_payload, state, gateway_receipt) + VALUES (?, ?, 1, ?, CAST(? AS jsonb), ?, '{}') + ON CONFLICT (pool_id, org_id) DO UPDATE SET + attempt = duckgres_trino_pool_publications.attempt + 1, + pending_intent = EXCLUDED.pending_intent, + pending_payload = EXCLUDED.pending_payload, + updated_at = now() + RETURNING attempt`, + poolID, orgID, kind, payload, TrinoPublicationPending).Scan(&attempt).Error + }) + return attempt, err +} + +// ResolveTrinoPoolPublicationIntent records that the open occurrence reached a +// DEFINITE outcome at the Gateway, so the next desired change takes a new one. +// +// It is deliberately separate from the checkpoint: the Gateway can answer +// definitively that an occurrence is spent WITHOUT this control plane knowing +// which body committed under it - a step recorded with a different intent says +// exactly that - and in that case the occurrence must be closed while the +// checkpoint must not move. +func (cs *ConfigStore) ResolveTrinoPoolPublicationIntent(ctx context.Context, lease TrinoPoolLease, poolID, orgID string) error { + if orgID == "" { + return errors.New("resolving an intent requires an org") + } + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Model(&TrinoPoolPublication{}). + Where("pool_id = ? AND org_id = ?", poolID, orgID). + Updates(map[string]any{ + "pending_intent": "", + "pending_payload": "{}", + "updated_at": time.Now().UTC(), + }).Error + }) +} + +// RecordTrinoPoolPublicationFailure records a tenant's failed attempt and the +// wait it earned, so one unserviceable warehouse cannot busy-loop or starve the +// tenants the driver would otherwise reach after it. +func (cs *ConfigStore) RecordTrinoPoolPublicationFailure(ctx context.Context, lease TrinoPoolLease, poolID, orgID string, nextAttemptAt time.Time, lastError string) error { + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Exec(` + INSERT INTO duckgres_trino_pool_publications + (pool_id, org_id, attempts, next_attempt_at, last_error, state, gateway_receipt) + VALUES (?, ?, 1, ?, ?, ?, '{}') + ON CONFLICT (pool_id, org_id) DO UPDATE SET + attempts = duckgres_trino_pool_publications.attempts + 1, + next_attempt_at = EXCLUDED.next_attempt_at, + last_error = EXCLUDED.last_error, + updated_at = now()`, + poolID, orgID, nextAttemptAt.UTC(), lastError, TrinoPublicationPending).Error + }) +} + +// ClearTrinoPoolPublicationFailure clears a tenant's backoff after a step that +// worked, so a tenant that recovers is not held behind a wait it no longer +// deserves. +func (cs *ConfigStore) ClearTrinoPoolPublicationFailure(ctx context.Context, lease TrinoPoolLease, poolID, orgID string) error { + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Model(&TrinoPoolPublication{}). + Where("pool_id = ? AND org_id = ? AND (attempts > 0 OR next_attempt_at IS NOT NULL)", poolID, orgID). + Updates(map[string]any{ + "attempts": 0, + "next_attempt_at": nil, + "last_error": "", + "updated_at": time.Now().UTC(), + }).Error + }) +} + +// RecordTrinoPoolPublicationCommitted checkpoints a COMMITTED barrier. +// +// The Gateway's record is authoritative from the moment it commits, so this is +// a checkpoint of something already true and never a retraction point: recovery +// after an interrupted checkpoint re-reads the Gateway and completes. +func (cs *ConfigStore) RecordTrinoPoolPublicationCommitted(ctx context.Context, lease TrinoPoolLease, poolID, orgID, targetRevision, receipt string) error { + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + if receipt == "" { + receipt = "{}" + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + result := tx.Model(&TrinoPoolPublication{}). + Where("pool_id = ? AND org_id = ?", poolID, orgID). + Updates(map[string]any{ + "admitted_target_revision": targetRevision, + "state": TrinoPublicationAdmitted, + "gateway_receipt": receipt, + // The barrier is finished, so it stops being the live attempt. + // Leaving it named here would select a committed publication as + // the one in flight forever: the driver would try to abandon it, + // the Gateway would refuse (an opened admission gate is never + // retracted), and the pass would re-checkpoint the same + // admission instead of reaching the next tenant. The outcome + // survives in admitted_target_revision and the receipt. + "publication_id": "", + "target_revision": "", + "last_error": "", + "updated_at": time.Now().UTC(), + }) + if result.Error != nil { + return result.Error + } + if result.RowsAffected != 1 { + return fmt.Errorf("%w: publication for org %q is unknown", ErrTrinoPoolConflict, orgID) + } + return nil + }) +} + +// RecordTrinoPoolTenantRevoked checkpoints a withdrawn admission. The row is +// kept: deleting it would read as "never published", and the next tick would +// republish the binding of a tenant that is meant to be gone. +func (cs *ConfigStore) RecordTrinoPoolTenantRevoked(ctx context.Context, lease TrinoPoolLease, poolID, orgID, reason string) error { + if poolID != lease.PoolID { + return fmt.Errorf("%w: publication belongs to pool %q", ErrTrinoPoolConflict, poolID) + } + return cs.withPoolAuthority(ctx, lease, func(tx *gorm.DB, _ *TrinoPool) error { + return tx.Model(&TrinoPoolPublication{}). + Where("pool_id = ? AND org_id = ?", poolID, orgID). + Updates(map[string]any{ + "state": TrinoPublicationRevoked, + "admitted_target_revision": "", + "publication_id": "", + "target_revision": "", + // The Gateway answered, so this occurrence is spent. + "pending_intent": "", + "pending_payload": "{}", + "last_error": reason, + "updated_at": time.Now().UTC(), + }).Error + }) +} + +// ListTrinoPoolPublications returns every tenant this pool has published, +// including revoked ones - which is what lets a tenant that disappeared from +// the projection be revoked exactly once rather than every tick. +func (cs *ConfigStore) ListTrinoPoolPublications(ctx context.Context, poolID string) ([]TrinoPoolPublication, error) { + var publications []TrinoPoolPublication + err := cs.db.WithContext(ctx).Where("pool_id = ?", poolID).Order("org_id").Find(&publications).Error + return publications, err +} diff --git a/controlplane/multitenant.go b/controlplane/multitenant.go index f0282b487..849ebb028 100644 --- a/controlplane/multitenant.go +++ b/controlplane/multitenant.go @@ -817,6 +817,18 @@ func SetupMultiTenant( } } + // Shared Trino compute pools. Constructed unconditionally so the wiring + // cannot rot while the feature is off: with no pooled cell in the registry + // this returns nothing and changes nothing. A wiring failure is fatal for + // the same reason the rest of the Trino branch is - an operator who + // configured a pool must not be left with a control plane that silently + // reconciles nothing. + poolOperators, poolErr := buildTrinoPoolOperators(store, trinoCells, cpInstanceID) + if poolErr != nil { + return nil, nil, nil, nil, nil, nil, fmt.Errorf("shared Trino pool wiring failed: %w", poolErr) + } + attachTrinoPoolOperators(janitorLeader, poolOperators) + if rolloutReadiness == nil { var rolloutErr error rolloutReadiness, rolloutErr = buildTrinoRolloutReadiness(trinoCells, store) diff --git a/controlplane/pre_ready_disconnect.go b/controlplane/pre_ready_disconnect.go index da9eb1ab9..bb171e316 100644 --- a/controlplane/pre_ready_disconnect.go +++ b/controlplane/pre_ready_disconnect.go @@ -79,10 +79,11 @@ func (w *preReadyDisconnectWatcher) watch() { } func (w *preReadyDisconnectWatcher) finish(result preReadyDisconnectResult) { + // Publish the result before cancellation lets session creation call Stop. + w.done <- result if result.ClientCanceled { w.cancel() } - w.done <- result } // Stop joins the watcher and clears the temporary read deadline before the diff --git a/controlplane/pre_ready_disconnect_test.go b/controlplane/pre_ready_disconnect_test.go index 3591cdeef..d79a6ba7a 100644 --- a/controlplane/pre_ready_disconnect_test.go +++ b/controlplane/pre_ready_disconnect_test.go @@ -41,6 +41,41 @@ func TestPreReadyDisconnectWatcherCancelsContextWhenClientCloses(t *testing.T) { } } +func TestPreReadyDisconnectWatcherPublishesResultBeforeCancellation(t *testing.T) { + cancelStarted := make(chan struct{}) + releaseCancel := make(chan struct{}) + finished := make(chan struct{}) + watcher := &preReadyDisconnectWatcher{ + done: make(chan preReadyDisconnectResult, 1), + cancel: func() { + close(cancelStarted) + <-releaseCancel + }, + } + defer func() { + close(releaseCancel) + <-finished + }() + want := preReadyDisconnectResult{ClientCanceled: true, Err: io.EOF} + go func() { + watcher.finish(want) + close(finished) + }() + select { + case <-cancelStarted: + case <-time.After(time.Second): + t.Fatal("watcher did not cancel session creation") + } + select { + case got := <-watcher.done: + if got != want { + t.Fatalf("published result = %+v, want %+v", got, want) + } + default: + t.Fatal("session cancellation became visible before its disconnect result") + } +} + func TestPreReadyDisconnectWatcherStopLeavesReaderUsable(t *testing.T) { clientConn, serverConn := net.Pipe() defer func() { _ = clientConn.Close() }() diff --git a/controlplane/provisioner/opa/builder.go b/controlplane/provisioner/opa/builder.go index 863b45883..5ba9fe673 100644 --- a/controlplane/provisioner/opa/builder.go +++ b/controlplane/provisioner/opa/builder.go @@ -2,9 +2,13 @@ package opa import ( "bytes" + "crypto/sha256" _ "embed" + "encoding/binary" + "encoding/hex" "encoding/json" "fmt" + "hash" "github.com/open-policy-agent/opa/v1/bundle" ) @@ -16,10 +20,10 @@ import ( //go:embed policy.rego var policyRego []byte -// bundleRevision is the manifest.revision stamped on bundles. OPA logs -// the revision on activation; bumping it on policy edits makes bundle -// pushes visible in OPA logs. The build encoding (data hash) further -// disambiguates bundles with the same revision but different data. +// bundleRevision is the schema prefix of every revision this package stamps. +// The revision itself is the prefix plus a digest of the projected data (see +// PolicyRevision): a constant could not tell a coordinator serving today's +// tenant set from one that predates it. const bundleRevision = "v2" // policyPath is the in-bundle path of policy.rego. The bundle library @@ -58,10 +62,25 @@ func (defaultBuilder) BuildBundle(gc GroupCatalogs, gs GroupScopes) ([]byte, err if err != nil { return nil, fmt.Errorf("build data document: %w", err) } + revision, err := PolicyRevision(gc, gs) + if err != nil { + return nil, err + } + // data.trino.revision is what a coordinator's OPA answers when it is asked + // which authorization data it decides with (`opa.policy.revision-uri`). + // Without this document OPA answers "undefined", the access control reports + // no loaded revision, and a controller has no way to tell a coordinator + // deciding with the current policy from one still serving a bundle from + // before a tenant existed. The manifest revision alone cannot do it: it is + // not queryable as a document. + data["trino"] = map[string]interface{}{"revision": revision} b := bundle.Bundle{ Manifest: bundle.Manifest{ - Revision: bundleRevision, + // The manifest revision carries the same value, so OPA's activation + // log names the exact projection it loaded rather than a constant + // that never changes. + Revision: revision, Roots: &[]string{"trino", "group_catalogs", "group_scopes"}, }, Modules: []bundle.ModuleFile{ @@ -87,6 +106,47 @@ func (defaultBuilder) BuildBundle(gc GroupCatalogs, gs GroupScopes) ([]byte, err return buf.Bytes(), nil } +// PolicyRevision is the revision stamped on the bundle built from exactly this +// projection. +// +// It is a digest of the policy and its data, not a counter, for two reasons: the +// producer is whichever control-plane replica serves the bundle, so no replica +// owns a counter, and the value has to be comparable in both directions - a +// controller asks "is the coordinator deciding with the data I currently +// serve?", which is an equality question, not an ordering one. +// +// The POLICY BYTES are part of it, not just the data. policy.rego is embedded +// in the CONTROL PLANE binary and served to OPA as a remote bundle, so it is +// not covered by the candidate's image check at all: two duckgres versions can +// serve different rules to the same Trino and OPA images with an identical +// group map. A revision over the data alone would call a coordinator deciding +// with the previous RULES current. +func PolicyRevision(gc GroupCatalogs, gs GroupScopes) (string, error) { + data, err := buildDataDocument(gc, gs) + if err != nil { + return "", fmt.Errorf("build data document: %w", err) + } + // Marshalling a map[string]interface{} sorts object keys, so the digest is + // stable across builds of the same projection. + canonical, err := json.Marshal(data) + if err != nil { + return "", fmt.Errorf("canonicalize bundle data: %w", err) + } + digest := sha256.New() + // Length-prefixed, so no rearrangement of policy and data bytes can produce + // the same digest as a different pair. + writeDigestField(digest, policyRego) + writeDigestField(digest, canonical) + return bundleRevision + "." + hex.EncodeToString(digest.Sum(nil)), nil +} + +func writeDigestField(digest hash.Hash, field []byte) { + var length [8]byte + binary.BigEndian.PutUint64(length[:], uint64(len(field))) + _, _ = digest.Write(length[:]) + _, _ = digest.Write(field) +} + // buildDataDocument builds the JSON-decoded map[string]interface{} that OPA // stores under data.. We always emit `group_catalogs` even when gc is // nil so the policy's `data.group_catalogs[group][catalog]` lookup is diff --git a/controlplane/provisioner/opa/builder_test.go b/controlplane/provisioner/opa/builder_test.go index 332bac16f..029fb351c 100644 --- a/controlplane/provisioner/opa/builder_test.go +++ b/controlplane/provisioner/opa/builder_test.go @@ -46,8 +46,12 @@ func TestBuildBundleRoundTrip(t *testing.T) { t.Fatalf("bundle.Read: %v", err) } - if parsed.Manifest.Revision != bundleRevision { - t.Errorf("manifest revision: want %q, got %q", bundleRevision, parsed.Manifest.Revision) + revision, err := PolicyRevision(gc, nil) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + if parsed.Manifest.Revision != revision { + t.Errorf("manifest revision: want %q, got %q", revision, parsed.Manifest.Revision) } // Policy file must be present and contain the package declaration. @@ -532,3 +536,104 @@ func TestNewGroupScopeDropsMalformedRelations(t *testing.T) { t.Errorf("relation_schemas = %v, want only ok", scope.RelationSchemas) } } + +// A coordinator answers "which authorization data am I deciding with?" out of +// its OPA, and OPA can only answer it from a document the bundle carries. A +// bundle without one leaves the access control unable to report anything, which +// is indistinguishable from a coordinator serving a projection that predates a +// tenant. +func TestBuildBundlePublishesItsRevisionAsADocument(t *testing.T) { + gc := GroupCatalogs{"org_42": {"org_42": true}} + + raw, err := NewBuilder().BuildBundle(gc, nil) + if err != nil { + t.Fatalf("BuildBundle: %v", err) + } + parsed, err := bundle.NewReader(bytes.NewReader(raw)).Read() + if err != nil { + t.Fatalf("bundle.Read: %v", err) + } + + trino, ok := parsed.Data["trino"].(map[string]interface{}) + if !ok { + t.Fatalf("data.trino is not a map: %T", parsed.Data["trino"]) + } + revision, ok := trino["revision"].(string) + if !ok || revision == "" { + t.Fatalf("data.trino.revision = %v, want the served revision", trino["revision"]) + } + want, err := PolicyRevision(gc, nil) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + if revision != want { + t.Errorf("data.trino.revision = %q, want %q", revision, want) + } + if parsed.Manifest.Revision != revision { + t.Errorf("manifest revision %q disagrees with the served document %q", parsed.Manifest.Revision, revision) + } +} + +// The revision has to change when the authorization data changes and stay put +// when it does not: a controller compares it for equality to decide whether a +// coordinator is current, so a constant would certify a stale one and a +// nondeterministic value would certify nothing at all. +func TestPolicyRevisionTracksTheProjection(t *testing.T) { + first, err := PolicyRevision(GroupCatalogs{"org_42": {"org_42": true}}, nil) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + same, err := PolicyRevision(GroupCatalogs{"org_42": {"org_42": true}}, nil) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + if first != same { + t.Errorf("the same projection produced %q and %q", first, same) + } + withTenant, err := PolicyRevision(GroupCatalogs{ + "org_42": {"org_42": true}, + "org_43": {"org_43": true}, + }, nil) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + if withTenant == first { + t.Error("adding a tenant did not change the revision") + } + scoped, err := PolicyRevision( + GroupCatalogs{"org_42": {"org_42": true}}, + GroupScopes{"scope_org_42_team_7": NewGroupScope([]string{"posthog"}, nil)}, + ) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + if scoped == first { + t.Error("adding a project scope did not change the revision") + } +} + +// policy.rego lives in the CONTROL PLANE binary and is served to OPA as a +// remote bundle, so the candidate's image check - which compares the Trino and +// OPA images - says nothing about which rules a coordinator is deciding with. +// Two control-plane versions can serve different rules with an identical group +// map, and a revision that covered only the data would call the older one +// current. +func TestPolicyRevisionCoversThePolicyBytes(t *testing.T) { + gc := GroupCatalogs{"org_42": {"org_42": true}} + baseline, err := PolicyRevision(gc, nil) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + + original := policyRego + t.Cleanup(func() { policyRego = original }) + policyRego = append(append([]byte{}, original...), []byte("\n# a rule change\n")...) + + changed, err := PolicyRevision(gc, nil) + if err != nil { + t.Fatalf("PolicyRevision after a policy change: %v", err) + } + if changed == baseline { + t.Fatal("the same data with different policy bytes produced the same revision") + } +} diff --git a/controlplane/provisioner/opa/policy_test.go b/controlplane/provisioner/opa/policy_test.go index 389e3a131..7054a92dd 100644 --- a/controlplane/provisioner/opa/policy_test.go +++ b/controlplane/provisioner/opa/policy_test.go @@ -2190,3 +2190,50 @@ func TestBatchedFilteringMatchesNonBatchedForScopes(t *testing.T) { } } } + +// The bundle publishes its revision as data.trino.revision, in the same +// document root the policy's package occupies. OPA refuses a bundle whose data +// collides with a rule path, and a refused bundle means the coordinator keeps +// authorizing with whatever it loaded last - so the coexistence is asserted +// against a real evaluation rather than assumed. +func TestPolicyEvaluatesAlongsideTheRevisionDocument(t *testing.T) { + ctx := context.Background() + gc := GroupCatalogs{"org_42": {"org_42": true}} + data, err := buildDataDocument(gc, nil) + if err != nil { + t.Fatalf("buildDataDocument: %v", err) + } + revision, err := PolicyRevision(gc, nil) + if err != nil { + t.Fatalf("PolicyRevision: %v", err) + } + data["trino"] = map[string]interface{}{"revision": revision} + + q, err := rego.New( + rego.Query("data.trino.allow"), + rego.Module("policy.rego", string(policyRego)), + rego.Data(data), + ).PrepareForEval(ctx) + if err != nil { + t.Fatalf("PrepareForEval with the revision document: %v", err) + } + if !evalAllow(t, q, buildInput("42", "ExecuteQuery", nil)) { + t.Error("a tenant's own catalog was denied once the revision document was present") + } + + served, err := rego.New( + rego.Query("data.trino.revision"), + rego.Module("policy.rego", string(policyRego)), + rego.Data(data), + ).PrepareForEval(ctx) + if err != nil { + t.Fatalf("PrepareForEval for the revision: %v", err) + } + results, err := served.Eval(ctx) + if err != nil { + t.Fatalf("eval revision: %v", err) + } + if len(results) != 1 || results[0].Expressions[0].Value != revision { + t.Fatalf("data.trino.revision did not answer with %q: %v", revision, results) + } +} diff --git a/controlplane/provisioner/opa/serve.go b/controlplane/provisioner/opa/serve.go index d0e27084c..f26002464 100644 --- a/controlplane/provisioner/opa/serve.go +++ b/controlplane/provisioner/opa/serve.go @@ -23,6 +23,19 @@ import ( type Bundle struct { bytes []byte ETag string + // Revision names the PROJECTION these bytes were built from, as the + // control plane's durable record names it. It is empty for a cell that + // does not fence its projection (every legacy cell), and the handler then + // serves exactly as before. + Revision string +} + +// WithRevision labels a bundle with the projection it was built from, so a +// serving replica can be asked whether that projection is still the accepted +// one before the bytes leave the process. +func (b Bundle) WithRevision(revision string) Bundle { + b.Revision = revision + return b } // NewBundle wraps a freshly built bundle in a Bundle, computing a strong @@ -120,6 +133,18 @@ func (s *BundleStore) Current() (Bundle, bool) { type Handler struct { Store *BundleStore Auth func(r *http.Request) bool + // AcceptedRevision reports the projection the control plane's durable + // record currently accepts, for deployments that fence it. + // + // Every replica builds and serves this bundle from its own view of the + // config store, so a replica whose view is behind would otherwise keep + // handing coordinators authorization data that has already been replaced - + // including, after a warehouse is removed, a roster that still contains it. + // An ETag cannot prevent that: it describes the bytes, not their age. + // + // Nil means the deployment does not fence its projection (every legacy + // cell), and serving is unchanged. + AcceptedRevision func() (string, bool) } // NewHandler constructs a bundle Handler. Both store and auth are @@ -213,6 +238,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "bundle not ready", http.StatusServiceUnavailable) return } + // The gate applies to the bundle CAPTURED above and to every answer derived + // from it. Two details are load-bearing: + // + // - It is checked before the 304 as well as the 200. A 304 means "keep + // what you have", so answering one from a projection this replica must + // not serve preserves exactly the stale authorization data the fence + // exists to retire. + // - Nothing re-reads the store after this point. Gating one bundle and + // serving whatever the store holds a moment later would check a + // different object than the one on the wire. + if h.AcceptedRevision != nil { + accepted, known := h.AcceptedRevision() + if !known || accepted == "" || accepted != b.Revision { + // This replica is not holding the accepted projection: it is behind, + // or ahead of what the authority has recorded. Either way the bytes + // stay here. OPA treats 503 as transient, keeps the bundle it already + // activated, and retries - which is the safe direction, because the + // alternative is replacing current authorization data with older. + http.Error(w, "bundle is not the accepted projection", http.StatusServiceUnavailable) + return + } + } w.Header().Set("Content-Type", "application/gzip") w.Header().Set("ETag", b.ETag) if match := r.Header.Get("If-None-Match"); match != "" && match == b.ETag { diff --git a/controlplane/provisioner/trino_cluster_secrets_test.go b/controlplane/provisioner/trino_cluster_secrets_test.go index 363597a5e..23a55db0e 100644 --- a/controlplane/provisioner/trino_cluster_secrets_test.go +++ b/controlplane/provisioner/trino_cluster_secrets_test.go @@ -4,6 +4,8 @@ package provisioner import ( "context" + "crypto/sha256" + "encoding/hex" "sync" "testing" @@ -498,3 +500,19 @@ func TestBootstrap_ObserverKeyLossRegenerates(t *testing.T) { t.Errorf("regenerated observer pair does not validate: %v", err) } } + +// The fingerprint a controller computes for a file it wrote must equal what +// Trino's file components report having loaded. Their published contract is +// `sha256:` plus the lower-case hex SHA-256 of the file's bytes; if this +// diverges, a pooled candidate can never be certified and nothing else would +// say why. +func TestTrinoFileFingerprintMatchesThePublishedContract(t *testing.T) { + digest := sha256.Sum256([]byte("alice:hash\n")) + want := "sha256:" + hex.EncodeToString(digest[:]) + if got := TrinoFileFingerprint([]byte("alice:hash\n")); got != want { + t.Fatalf("fingerprint = %q, want %q", got, want) + } + if TrinoFileFingerprint([]byte("alice:hash\n")) == TrinoFileFingerprint([]byte("alice:other\n")) { + t.Fatal("two different files produced the same fingerprint") + } +} diff --git a/controlplane/provisioner/trino_hoglake.go b/controlplane/provisioner/trino_hoglake.go index d22fb15ef..4593ed9ab 100644 --- a/controlplane/provisioner/trino_hoglake.go +++ b/controlplane/provisioner/trino_hoglake.go @@ -183,6 +183,20 @@ func isManagedHoglake(org configstore.TrinoEnabledOrg) bool { return configstore.EffectiveTrinoBackend(org.Backend) == configstore.TrinoBackendHoglake } +// ManagedHoglakeConfigured reports whether this provisioner can provision a +// managed Hoglake tenant at all: the service configuration AND the storage +// resolver that supplies the tenant's IAM role and region. +// +// It is exported for the startup wiring's own test. A cell that silently lost +// either input builds and reconciles perfectly until the first Hoglake tenant +// is provisioned, and then holds that warehouse pending with an error about +// configuration nobody changed - so "the pooled branch passes the same inputs +// as the legacy one" is worth asserting at the boundary rather than +// discovering per tenant. +func (p *TrinoProvisioner) ManagedHoglakeConfigured() bool { + return p.managedHoglake != nil && p.hoglakeDucklings != nil +} + func (p *TrinoProvisioner) managedHoglakeProperties(orgID string, d *DucklingStatus) (map[string]string, error) { if p.managedHoglake == nil { return nil, errors.New("managed Hoglake is not configured") diff --git a/controlplane/provisioner/trino_projection_fence_test.go b/controlplane/provisioner/trino_projection_fence_test.go new file mode 100644 index 000000000..191badc16 --- /dev/null +++ b/controlplane/provisioner/trino_projection_fence_test.go @@ -0,0 +1,142 @@ +//go:build kubernetes + +package provisioner + +import ( + "context" + "errors" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/provisioner/opa" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubefake "k8s.io/client-go/kubernetes/fake" +) + +// recordedFence is the durable record, with the accepted projection and this +// process's right to advance it both under the test's control. +type recordedFence struct { + advanceable bool + accepted string + acceptedRevision int64 + acceptedCalls int +} + +func (f *recordedFence) Accept(_ context.Context, build func([]configstore.TrinoEnabledOrg) (string, error)) (int64, error) { + if !f.advanceable { + return 0, ErrTrinoProjectionNotAdvanceable + } + digest, err := build(nil) + if err != nil { + return 0, err + } + f.accepted, f.acceptedRevision = digest, f.acceptedRevision+1 + return f.acceptedRevision, nil +} + +func (f *recordedFence) Accepted(context.Context) (string, int64, bool) { + f.acceptedCalls++ + return f.accepted, f.acceptedRevision, f.accepted != "" +} + +func fencedProvisioner(t *testing.T) *TrinoProvisioner { + t.Helper() + return &TrinoProvisioner{ + kubernetes: kubefake.NewClientset(), + namespace: "fence-test", + bundleStore: &opa.BundleStore{}, + bundleBuilder: opa.NewBuilder(), + } +} + +func (p *TrinoProvisioner) authSecretRevision(t *testing.T) (string, bool) { + t.Helper() + secret, err := p.kubernetes.CoreV1().Secrets(p.namespace).Get( + context.Background(), TrinoAuthSecretName, metav1.GetOptions{}) + if err != nil { + return "", false + } + return secret.Annotations[TrinoProjectionRevisionAnnotation], true +} + +// A replica that may not ADVANCE the projection must still build and SERVE it. +// +// Every replica but one is in that state at any moment: they all answer the +// bundle endpoint, so a replica that refused to build would strand every +// coordinator polling it on its last-good bundle, and reporting "not the +// advancer" as a reconcile failure would mark every pooled warehouse Failed +// everywhere except on the leader. What such a replica must NOT do is publish: +// the auth Secret is written only when what it built is what the record has +// accepted. +func TestANonAdvancingReplicaServesButPublishesNothingUnaccepted(t *testing.T) { + t.Run("behind the accepted projection", func(t *testing.T) { + provisioner := fencedProvisioner(t) + fence := &recordedFence{accepted: "a projection this replica did not build", acceptedRevision: 7} + provisioner.SetProjectionFence(fence) + + if err := provisioner.reconcileFencedProjection(context.Background(), nil); err != nil { + t.Fatalf("a replica that may not advance reported a reconcile failure: %v", err) + } + if _, serving := provisioner.bundleStore.Current(); !serving { + t.Fatal("the replica served no bundle, stranding the coordinators that poll it") + } + if _, written := provisioner.authSecretRevision(t); written { + t.Fatal("a projection the record has not accepted was published to the auth Secret") + } + }) + + t.Run("holding exactly the accepted projection", func(t *testing.T) { + advancing := fencedProvisioner(t) + accepting := &recordedFence{advanceable: true} + advancing.SetProjectionFence(accepting) + if err := advancing.reconcileFencedProjection(context.Background(), nil); err != nil { + t.Fatalf("the advancing replica failed: %v", err) + } + if revision, written := advancing.authSecretRevision(t); !written || revision != "1" { + t.Fatalf("auth secret revision = %q written=%v, want the revision the fence allocated", revision, written) + } + + // A second replica builds the SAME bytes from the same sources. It may + // not advance anything, but what it holds is what the pool accepted, so + // it projects it under that accepted revision. + follower := fencedProvisioner(t) + follower.SetProjectionFence(&recordedFence{accepted: accepting.accepted, acceptedRevision: accepting.acceptedRevision}) + if err := follower.reconcileFencedProjection(context.Background(), nil); err != nil { + t.Fatalf("the follower failed: %v", err) + } + if revision, written := follower.authSecretRevision(t); !written || revision != "1" { + t.Fatalf("follower auth secret revision = %q written=%v, want the accepted revision", revision, written) + } + }) + + t.Run("an unreadable record", func(t *testing.T) { + provisioner := fencedProvisioner(t) + provisioner.SetProjectionFence(&recordedFence{}) + + if err := provisioner.reconcileFencedProjection(context.Background(), nil); err != nil { + t.Fatalf("an unreadable record was reported as a reconcile failure: %v", err) + } + if _, serving := provisioner.bundleStore.Current(); !serving { + t.Fatal("the replica stopped serving because the record could not be read") + } + if _, written := provisioner.authSecretRevision(t); written { + t.Fatal("the auth Secret was written without knowing what the pool accepts") + } + }) + + t.Run("a genuine failure is still a failure", func(t *testing.T) { + provisioner := fencedProvisioner(t) + provisioner.SetProjectionFence(&failingFence{}) + if err := provisioner.reconcileFencedProjection(context.Background(), nil); err == nil { + t.Fatal("a fence that could not be consulted was treated as a routine refusal") + } + }) +} + +type failingFence struct{} + +func (failingFence) Accept(context.Context, func([]configstore.TrinoEnabledOrg) (string, error)) (int64, error) { + return 0, errors.New("the desired publisher could not be read") +} + +func (failingFence) Accepted(context.Context) (string, int64, bool) { return "", 0, false } diff --git a/controlplane/provisioner/trino_provisioner.go b/controlplane/provisioner/trino_provisioner.go index ec1073751..8a1fe0355 100644 --- a/controlplane/provisioner/trino_provisioner.go +++ b/controlplane/provisioner/trino_provisioner.go @@ -4,8 +4,10 @@ package provisioner import ( "context" + "crypto/sha256" "crypto/tls" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -18,6 +20,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/google/uuid" @@ -227,6 +230,17 @@ func projectableTrinoUsername(username string) bool { return len(username) <= 255 && trinoUsernamePattern.MatchString(username) } +// ProjectableTrinoUsername reports whether a username reaches the coordinator's +// password file at all. +// +// It is exported because the pooled admission binding must publish EXACTLY the +// principals password.db contains: a username this refuses never authenticates, +// so binding it would advertise a principal Trino rejects, and deriving the two +// sets from different code is how they drift apart. +func ProjectableTrinoUsername(username string) bool { + return projectableTrinoUsername(username) +} + // TrinoScopeGroupName returns the group label for a project-scoped login: // one group per (org, team), carrying that team's schema scope in the OPA // bundle. @@ -251,6 +265,31 @@ func TrinoResourceGroupName(principal string) string { return "root.tenants." + trinoSanitize(principal) } +// ErrTrinoCatalogNotThisReplica marks a catalog client that cannot answer on +// THIS control plane, as opposed to one that failed. +// +// A shared-pool cell publishes catalogs through a fence that only the replica +// holding the pool authority owns. The provisioning controller runs on every +// replica, so on all the others every catalog call must refuse - but that +// refusal is not a statement about the warehouse. Attributing it per org marked +// every pooled tenant Failed every tick and flapped ready_at/failed_at on rows +// that were serving perfectly. A reconcile that hits this leaves the Trino +// state rows untouched instead. +var ErrTrinoCatalogNotThisReplica = errors.New("this control plane does not own the catalog write path") + +// ErrTrinoNodeInventoryUnavailable marks a catalog client that cannot answer +// "which nodes are in this cluster" because there is no ONE cluster to ask. +// +// A pooled cell publishes catalogs to the shared store and its compute is a set +// of instances that come and go, so the legacy readiness probe - which asks a +// fixed coordinator for its node inventory and then checks every tenant's +// mounted credential on those nodes - has no coordinator to address. Its +// evidence comes from the pool instead: every member proves it applied the +// catalog and the projections before it is admitted, and (with the Gateway's +// admission restriction on) a tenant is only queryable once every serving +// member has acknowledged it. +var ErrTrinoNodeInventoryUnavailable = errors.New("this catalog client has no single coordinator to read a node inventory from") + // TrinoCatalogClient is the REST surface the provisioner needs against // the customer Trino cluster: enumerate, create, alter, drop catalogs. // Concrete implementation in trinoCatalogHTTPClient below; the interface @@ -444,29 +483,44 @@ type TrinoDucklingResolver func(ctx context.Context, orgID string) (*DucklingSta // fires on first install; thereafter ensureClusterSecrets adopts the // existing K8s Secrets. type TrinoProvisioner struct { - managed *TrinoManagedCatalogOpts - store TrinoStore - bootstrapSentinel TrinoBootstrapSentinelStore - warehouses TrinoWarehouseStore - ducklings TrinoDucklingResolver - hoglakeDucklings TrinoDucklingResolver - kubernetes kubernetes.Interface - secretReadiness TrinoSecretReadiness - authReadiness TrinoAuthenticationReadiness - namespace string - cellID string - explicitAssignmentOnly bool - catalog TrinoCatalogClient + managed *TrinoManagedCatalogOpts + store TrinoStore + bootstrapSentinel TrinoBootstrapSentinelStore + warehouses TrinoWarehouseStore + ducklings TrinoDucklingResolver + hoglakeDucklings TrinoDucklingResolver + kubernetes kubernetes.Interface + secretReadiness TrinoSecretReadiness + authReadiness TrinoAuthenticationReadiness + namespace string + cellID string + explicitAssignmentOnly bool + catalog TrinoCatalogClient + // tenantAdmission reports whether a tenant's publication has committed on + // the pool that serves it. Nil everywhere except a shared-pool cell with the + // Gateway's admission restriction enabled. + tenantAdmission TenantAdmissionGate additionalCatalogs []TrinoCatalogClient catalogTimeout time.Duration existingInternalSecrets []string bundleStore *opa.BundleStore bundleBuilder opa.BundleBuilder - tenantSecretMountPath string - awsRegion string - s3MaxConnections int - filesystemCacheEnabled bool - managedHoglake *TrinoManagedHoglakeConfig + // policyRevision is the revision of the authorization projection currently + // served. It is read from the pool operator's validation goroutine while + // the reconcile loop writes it, so it is atomic rather than a plain field. + policyRevision atomic.Pointer[string] + // authRevisions are the fingerprints of the projected password and group + // files, read from the same goroutine and for the same reason. + authRevisions atomic.Pointer[trinoAuthRevisions] + // projectionFence builds and accepts the projection in one transaction, + // returning the revision that acceptance allocated. Nil for every cell that + // does not fence its projection, where these writes are unchanged. + projectionFence TrinoProjectionFence + tenantSecretMountPath string + awsRegion string + s3MaxConnections int + filesystemCacheEnabled bool + managedHoglake *TrinoManagedHoglakeConfig // adminPasswordHash is cached on each Reconcile from the // trino-auth K8s Secret and prepended to password.db on projection. @@ -618,6 +672,74 @@ func NewTrinoProvisioner(opts TrinoProvisionerOpts) (*TrinoProvisioner, error) { // startup logging and tests. func (p *TrinoProvisioner) CellID() string { return p.cellID } +// SetCatalogClient replaces the catalog write path at runtime. +// +// A shared-pool cell has no fixed coordinator to issue CREATE CATALOG against, +// so its catalogs are published directly to the shared store. The publisher can +// only be built once a control plane holds the pool authority, which happens +// after startup - hence a setter rather than a constructor argument. Everything +// else about the reconcile loop is unchanged. +func (p *TrinoProvisioner) SetCatalogClient(catalog TrinoCatalogClient) { + p.credMu.Lock() + defer p.credMu.Unlock() + p.catalog = catalog +} + +// TenantAdmissionGate answers whether a tenant is ADMITTED on the pool that +// serves it - that is, whether its publication barrier has committed, so the +// Gateway will actually dispatch its queries. The string is the operator-facing +// reason when it is not. +type TenantAdmissionGate func(orgID string) (admitted bool, reason string) + +// SetTenantAdmissionGate installs that check. It is set only for a shared-pool +// cell that has the Gateway's admission restriction enabled; everywhere else it +// stays nil and nothing changes. +func (p *TrinoProvisioner) SetTenantAdmissionGate(gate TenantAdmissionGate) { + p.credMu.Lock() + defer p.credMu.Unlock() + p.tenantAdmission = gate +} + +func (p *TrinoProvisioner) tenantAdmissionGate() TenantAdmissionGate { + p.credMu.RLock() + defer p.credMu.RUnlock() + return p.tenantAdmission +} + +// applyTenantAdmissionGate holds a warehouse at Provisioning until its tenant is +// actually admitted. +// +// With the Gateway's admission restriction on, a catalog that exists and a +// coordinator that is healthy are NOT enough: the Gateway refuses to dispatch +// work for a tenant whose publication has not committed, so reporting Ready +// would tell an operator - and PostHog - that a warehouse is queryable when +// every query it receives is refused. +func (p *TrinoProvisioner) applyTenantAdmissionGate(outcomes map[string]catalogOutcome) { + gate := p.tenantAdmissionGate() + if gate == nil { + return + } + for orgID, outcome := range outcomes { + if outcome.Err != nil || outcome.Pending { + continue + } + if admitted, reason := gate(orgID); !admitted { + if reason == "" { + reason = "waiting for the pool to admit this warehouse" + } + outcomes[orgID] = catalogOutcome{Pending: true, PendingReason: reason} + } + } +} + +// catalogClient reads the current write path under the same lock the setter +// takes, so a reconcile tick cannot observe a half-installed client. +func (p *TrinoProvisioner) catalogClient() TrinoCatalogClient { + p.credMu.RLock() + defer p.credMu.RUnlock() + return p.catalog +} + // Reconcile runs one full projection: cluster secrets → auth files → // resource groups → OPA bundle → tenant passwords → catalogs. Errors in // any one output are logged and surfaced but the next output still runs — @@ -689,12 +811,27 @@ func (p *TrinoProvisioner) Reconcile(ctx context.Context) error { projectable, collisions := rejectPrincipalCollisions(orgs) var errs []error + // fencedProjection records that the authorization and authentication + // projections were handled together under the projection fence, so the + // separate OPA-bundle step does not rebuild them from a different read. + fencedProjection := false // 1. Auth file projection (K8s Secret). Atomic Secret update. // Runs BEFORE catalogs so the admin lines exist in password.db // + group.db when the catalog client's first request reaches // the coordinator on a cold-start tick. - authErr := p.reconcileAuthSecret(ctx, projectable) + // A FENCED cell (a pooled one) does this differently: the authorization + // and authentication projections are built from rows read inside the + // transaction that accepts them, and written under the revision that + // transaction allocated - so a replica can never stamp bytes it built + // from some earlier view with a number it read later. It also covers + // step 3 below, because the two projections are accepted together. + authErr := p.reconcileFencedProjection(ctx, projectable) + if authErr == errTrinoProjectionNotFenced { + authErr = p.reconcileAuthSecret(ctx, projectable) + } else { + fencedProjection = true + } if authErr != nil { errs = append(errs, fmt.Errorf("reconcile auth secret: %w", authErr)) } @@ -712,9 +849,12 @@ func (p *TrinoProvisioner) Reconcile(ctx context.Context) error { // the in-memory store the bundle HTTP handler serves. Pre- // catalog so the OPA sidecar's authorization decisions for the // catalog reconcile's own queries see the up-to-date roster. - opaErr := p.reconcileOPABundle(ctx, projectable) - if opaErr != nil { - errs = append(errs, fmt.Errorf("reconcile opa bundle: %w", opaErr)) + var opaErr error + if !fencedProjection { + opaErr = p.reconcileOPABundle(ctx, projectable) + if opaErr != nil { + errs = append(errs, fmt.Errorf("reconcile opa bundle: %w", opaErr)) + } } // 4. Tenant metadata-store passwords (K8s Secret). One key per @@ -775,6 +915,14 @@ func (p *TrinoProvisioner) Reconcile(ctx context.Context) error { catalogOutcomes, catErr = p.reconcileCatalogs(ctx, projectable, tenants) } if catErr != nil { + if errors.Is(catErr, ErrTrinoCatalogNotThisReplica) { + // Not this replica's work. Nothing failed, nothing is reported, + // and above all nothing is written: the owning replica keeps the + // state rows current. + slog.Debug("trino reconcile: catalog step belongs to another control plane", + "reason", catErr) + return errors.Join(errs...) + } errs = append(errs, fmt.Errorf("reconcile catalogs: %w", catErr)) } } else { @@ -787,6 +935,12 @@ func (p *TrinoProvisioner) Reconcile(ctx context.Context) error { // since each is a single K8s API write — wrap them once. Per-org // variance lives at the catalog step, which folds in the per-org // tenant-password outcomes. + // A tenant the pool has not admitted is not queryable, whatever its catalog + // says. This runs before the state writes so such a warehouse reads as + // Provisioning rather than Ready. + if catalogOutcomes != nil { + p.applyTenantAdmissionGate(catalogOutcomes) + } if len(collisions) > 0 { if catalogOutcomes == nil { catalogOutcomes = make(map[string]catalogOutcome, len(collisions)) @@ -1115,7 +1269,7 @@ func (p *TrinoProvisioner) ensureClusterSecrets(ctx context.Context) (bundleToke p.setObserverCredential(observerPlaintext, observerHash) // Push the admin plaintext into the catalog client if it supports // runtime credential updates (test fakes don't). - if updater, ok := p.catalog.(TrinoCatalogCredentialUpdater); ok { + if updater, ok := p.catalogClient().(TrinoCatalogCredentialUpdater); ok { updater.SetCredentials(opa.AdminPrincipal, adminPlaintext) } for _, catalog := range p.additionalCatalogs { @@ -1631,7 +1785,7 @@ func (p *TrinoProvisioner) reconcileCatalogs( orgs []configstore.TrinoEnabledOrg, tenants tenantSecretProjection, ) (map[string]catalogOutcome, error) { - outcomes, firstErr := p.reconcileBoundedBackend(ctx, orgs, tenants, p.catalog, "primary") + outcomes, firstErr := p.reconcileBoundedBackend(ctx, orgs, tenants, p.catalogClient(), "primary") errs := []error{firstErr} for i, catalog := range p.additionalCatalogs { backendOutcomes, err := p.reconcileBoundedBackend(ctx, orgs, tenants, catalog, fmt.Sprintf("additional-%d", i)) @@ -1665,6 +1819,16 @@ func (p *TrinoProvisioner) reconcileBoundedBackend(ctx context.Context, orgs []c } pending, readinessErr := p.reconcileBackendReadiness(backendCtx, catalog, expected, backend) + if errors.Is(readinessErr, ErrTrinoNodeInventoryUnavailable) { + // A pooled cell has no single coordinator to take an inventory from, and + // inventing one would be worse than skipping: its readiness evidence is + // the pool's own admission, where each member proves it applied the + // catalog and the projections. Failing every tenant on a probe that + // cannot apply to them would mark the whole pool broken. + slog.Debug("trino reconcile: skipping the fixed-coordinator readiness probe for a pooled cell", + "reason", readinessErr) + return outcomes, catalogErr + } for org := range expected { if readinessErr != nil { outcomes[org] = catalogOutcome{Err: readinessErr} @@ -2127,6 +2291,9 @@ func (p *TrinoProvisioner) reconcileAuthSecret(ctx context.Context, orgs []confi AdminPasswordHash: p.adminPasswordHash, ObserverPasswordHash: p.observerHash(), }) + // The unfenced path, unchanged: no projection revision is stamped and the + // write is the ordinary merge. A fenced cell does not come through here at + // all - see reconcileFencedProjection. files := map[string][]byte{ TrinoAuthSecretKeyPasswordDB: []byte(passwordDB), TrinoAuthSecretKeyGroupDB: []byte(groupDB), @@ -2137,9 +2304,219 @@ func (p *TrinoProvisioner) reconcileAuthSecret(ctx context.Context, orgs []confi // Update even when no tenant catalogs remain: disabling must invalidate // any previous authentication observation before a later re-enable. p.authReadiness.SetExpected(files) + // The fingerprints of the bytes just projected. A pooled candidate must + // report loading exactly these before it may be admitted: its OPA bundle + // and its password file arrive by different paths and at different times, + // so a coordinator can be current on authorization data while its password + // store still predates the tenant that is about to be admitted - it would + // pass an authorization-only check and then reject that tenant's first + // request. + p.authRevisions.Store(&trinoAuthRevisions{ + Password: TrinoFileFingerprint([]byte(passwordDB)), + Group: TrinoFileFingerprint([]byte(groupDB)), + }) + return nil +} + +// trinoAuthRevisions are the fingerprints of the projected authentication +// files, held together so a reader can never pair one file's fingerprint with +// the other's projection. +type trinoAuthRevisions struct { + Password string + Group string +} + +// TrinoFileFingerprint computes what Trino's file password authenticator and +// file group provider report as their loaded revision. +// +// This is their PUBLISHED contract - `sha256:` followed by the lower-case +// hexadecimal SHA-256 of the file's bytes - so a controller that wrote the file +// can compute the value it expects to see acknowledged without asking a +// coordinator what its revision means. +func TrinoFileFingerprint(content []byte) string { + digest := sha256.Sum256(content) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +// errTrinoProjectionNotFenced means this cell does not fence its projection - +// every legacy cell - so the caller runs the ordinary projection steps. +var errTrinoProjectionNotFenced = errors.New("this cell does not fence its projection") + +// trinoProjection is one coherent set of projected bytes: the authentication +// files and the authorization bundle built from ONE read of the source rows, +// together with the digest that identifies them. +type trinoProjection struct { + passwordDB string + groupDB string + bundle []byte + policyRevision string + digest string +} + +// reconcileFencedProjection builds and publishes the projection under the +// control plane's durable projection fence. +// +// Two properties the unfenced path does not have: +// +// - the bytes and the revision describe the SAME read. The source rows are +// read inside the transaction that allocates the revision, and the bytes +// written afterwards are the ones built from those rows - not a buffer +// computed earlier and stamped with a number read later. +// - only the process the deployment currently wants may advance it. An older +// control plane that wins the lease publishes its own older policy rules, +// and a counter cannot notice that: it orders acceptances, not rule sets. +// +// When the fence refuses - this replica is not the desired publisher, or the +// projection has moved on - nothing is written and nothing is served. The +// coordinators keep the authorization data they already have. +func (p *TrinoProvisioner) reconcileFencedProjection(ctx context.Context, orgs []configstore.TrinoEnabledOrg) error { + p.credMu.RLock() + fence := p.projectionFence + p.credMu.RUnlock() + if fence == nil { + return errTrinoProjectionNotFenced + } + + var built trinoProjection + revision, err := fence.Accept(ctx, func(sourceOrgs []configstore.TrinoEnabledOrg) (string, error) { + projectable, _ := rejectPrincipalCollisions(sourceOrgs) + projection, buildErr := p.buildProjection(projectable) + if buildErr != nil { + return "", buildErr + } + built = projection + return projection.digest, nil + }) + switch { + case err == nil: + // This replica advanced the projection, and `built` is the projection + // the accepted revision was allocated for - same read, same bytes. + case errors.Is(err, ErrTrinoProjectionNotAdvanceable): + // Every replica but one is here at any moment. It still BUILDS and + // SERVES the projection - refusing to would leave the coordinators that + // poll this replica on their last-good bundle forever - but it publishes + // nothing the record has not accepted. + built, err = p.buildProjection(orgs) + if err != nil { + return err + } + accepted, acceptedRevision, ok := fence.Accepted(ctx) + if !ok { + // The record cannot be read, so nothing can be said about what this + // replica holds. It keeps serving what it has; the gate on the + // bundle endpoint makes the same call independently. + p.publishProjectionLocally(built) + return nil + } + if accepted != built.digest { + // Behind, or ahead of what has been accepted. Serve nothing new and + // touch no Secret: the gate will refuse these bytes anyway, and + // writing them would be the regression the fence exists to prevent. + p.publishProjectionLocally(built) + return nil + } + revision = acceptedRevision + default: + return fmt.Errorf("accept the authorization projection: %w", err) + } + if built.digest == "" { + return errors.New("the projection fence accepted nothing to publish") + } + + if err := p.upsertSecretMerge(ctx, TrinoAuthSecretName, map[string][]byte{ + TrinoAuthSecretKeyPasswordDB: []byte(built.passwordDB), + TrinoAuthSecretKeyGroupDB: []byte(built.groupDB), + }, revision); err != nil { + return err + } + p.publishProjectionLocally(built) return nil } +// publishProjectionLocally records what this process is serving. It does not +// decide whether those bytes may leave the process - the accepted record does, +// through the bundle handler's gate. +func (p *TrinoProvisioner) publishProjectionLocally(built trinoProjection) { + p.authRevisions.Store(&trinoAuthRevisions{ + Password: TrinoFileFingerprint([]byte(built.passwordDB)), + Group: TrinoFileFingerprint([]byte(built.groupDB)), + }) + policyRevision := built.policyRevision + p.policyRevision.Store(&policyRevision) + p.bundleStore.Set(opa.NewBundle(built.bundle).WithRevision(built.digest)) +} + +// buildProjection renders both projections from ONE set of source rows and +// fingerprints exactly the bytes it produced. +func (p *TrinoProvisioner) buildProjection(orgs []configstore.TrinoEnabledOrg) (trinoProjection, error) { + passwordDB, groupDB := BuildTrinoAuthFiles(orgs, TrinoClusterPrincipals{ + AdminPasswordHash: p.adminPasswordHash, + ObserverPasswordHash: p.observerHash(), + }) + gc, gs := p.authorizationDocuments(orgs) + bundle, err := p.bundleBuilder.BuildBundle(gc, gs) + if err != nil { + return trinoProjection{}, fmt.Errorf("build opa bundle: %w", err) + } + policyRevision, err := opa.PolicyRevision(gc, gs) + if err != nil { + return trinoProjection{}, fmt.Errorf("compute opa policy revision: %w", err) + } + projection := trinoProjection{ + passwordDB: passwordDB, groupDB: groupDB, bundle: bundle, policyRevision: policyRevision, + } + // The digest fingerprints the bytes this call produced, not whatever the + // provisioner happens to hold later: that is what makes the revision the + // store allocates describe THESE bytes. + projection.digest = TrinoProjectionDigest(policyRevision, + TrinoFileFingerprint([]byte(passwordDB)), TrinoFileFingerprint([]byte(groupDB))) + return projection, nil +} + +// TrinoProjectionDigest names one coherent projection: the authorization +// bundle's revision and the fingerprints of the two authentication files. +// +// Exported because it is the SAME function on both sides of the fence: the +// control plane digests what it published, and a candidate's admission digests +// what that coordinator reports having loaded. Two implementations of this +// would be two definitions of "the same projection". +func TrinoProjectionDigest(policyRevision, passwordFingerprint, groupFingerprint string) string { + if policyRevision == "" || passwordFingerprint == "" || groupFingerprint == "" { + return "" + } + digest := sha256.Sum256([]byte(strings.Join( + []string{policyRevision, passwordFingerprint, groupFingerprint}, "\x00"))) + return hex.EncodeToString(digest[:]) +} + +// ProjectionDigest names the authorization and authentication data this +// control plane has projected, as one value: the OPA bundle's revision and the +// fingerprints of the password and group files. +// +// It is what the durable projection fence is keyed on. It is empty until all +// three have been produced, which is the honest answer for a replica that has +// not finished a projection yet - and an empty digest matches no accepted +// projection, so such a replica serves nothing. +func (p *TrinoProvisioner) ProjectionDigest() string { + password, group := p.PublishedAuthRevisions() + policy := p.PublishedPolicyRevision() + if policy == "" || password == "" || group == "" { + return "" + } + return TrinoProjectionDigest(policy, password, group) +} + +// PublishedAuthRevisions reports the fingerprints of the authentication files +// this control plane has projected, or ("", "") before the first projection. +// Empty means nothing may be claimed on their behalf, which fails a pooled +// admission closed. +func (p *TrinoProvisioner) PublishedAuthRevisions() (password, group string) { + if revisions := p.authRevisions.Load(); revisions != nil { + return revisions.Password, revisions.Group + } + return "", "" +} + // TrinoClusterPrincipals carries the bcrypt hashes for the cell's two // non-tenant principals. A struct rather than two positional strings // because they are the same type, adjacent, and swapping them would hand @@ -2640,6 +3017,34 @@ func BuildTrinoResourceGroups() ([]byte, error) { // in-memory), but kept on the signature for parity with the other // reconcile* steps and to permit instrumented builders later. func (p *TrinoProvisioner) reconcileOPABundle(_ context.Context, orgs []configstore.TrinoEnabledOrg) error { + gc, gs := p.authorizationDocuments(orgs) + bundle, err := p.bundleBuilder.BuildBundle(gc, gs) + if err != nil { + return fmt.Errorf("build opa bundle: %w", err) + } + // The revision of the projection now being served. A pooled candidate is + // only certified once its OPA reports deciding with THIS value: a + // structurally healthy coordinator whose policy engine still serves the + // previous projection would authorize against a tenant set that no longer + // exists. It is recorded before the bundle is published, so the value a + // reader sees is never newer than what is on the wire. + revision, err := opa.PolicyRevision(gc, gs) + if err != nil { + return fmt.Errorf("compute opa policy revision: %w", err) + } + p.policyRevision.Store(&revision) + // The bundle carries the PROJECTION it was built from, so a replica can be + // asked whether that projection is still the accepted one before these + // bytes leave the process. Empty for a cell that does not fence its + // projection, where the handler serves exactly as before. + p.bundleStore.Set(opa.NewBundle(bundle).WithRevision(p.ProjectionDigest())) + return nil +} + +// authorizationDocuments renders the bundle's data documents from one set of +// source rows. Shared by the fenced and unfenced paths so the two can never +// authorize differently for the same input. +func (p *TrinoProvisioner) authorizationDocuments(orgs []configstore.TrinoEnabledOrg) (opa.GroupCatalogs, opa.GroupScopes) { gc := make(opa.GroupCatalogs, len(orgs)+1) gs := opa.GroupScopes{} adminCatalogs := make(map[string]bool, len(orgs)) @@ -2671,12 +3076,18 @@ func (p *TrinoProvisioner) reconcileOPABundle(_ context.Context, orgs []configst // docstring). gc[opa.AdminGroup] = adminCatalogs } - bundle, err := p.bundleBuilder.BuildBundle(gc, gs) - if err != nil { - return fmt.Errorf("build opa bundle: %w", err) + return gc, gs +} + +// PublishedPolicyRevision is the authorization-data revision this control plane +// currently serves, or "" before the first projection. It is the value a +// candidate's OPA must report before the auth-revision check may be claimed; +// empty means nothing may claim it, which fails admission closed. +func (p *TrinoProvisioner) PublishedPolicyRevision() string { + if revision := p.policyRevision.Load(); revision != nil { + return *revision } - p.bundleStore.Set(opa.NewBundle(bundle)) - return nil + return "" } // upsertSecretMerge is the partial-owner Secret writer: it @@ -2689,7 +3100,13 @@ func (p *TrinoProvisioner) reconcileOPABundle(_ context.Context, orgs []configst // Deterministic projections retry on the next reconcile tick. // Credential establishment uses ensureCredentialPair's conditional snapshot // update instead: this helper must not overwrite a concurrent credential winner. -func (p *TrinoProvisioner) upsertSecretMerge(ctx context.Context, name string, data map[string][]byte) error { +// projectionRevisions is the optional fence: `revision` is the accepted +// projection this write belongs to (0 = the deployment does not fence). +func (p *TrinoProvisioner) upsertSecretMerge(ctx context.Context, name string, data map[string][]byte, projectionRevision ...int64) error { + var revision int64 + if len(projectionRevision) != 0 { + revision = projectionRevision[0] + } secrets := p.kubernetes.CoreV1().Secrets(p.namespace) existing, err := secrets.Get(ctx, name, metav1.GetOptions{}) if err != nil { @@ -2705,6 +3122,7 @@ func (p *TrinoProvisioner) upsertSecretMerge(ctx context.Context, name string, d "app": "trino", "duckgres/managed": "true", }, + Annotations: projectionAnnotations(revision), }, Type: corev1.SecretTypeOpaque, Data: data, @@ -2723,6 +3141,18 @@ func (p *TrinoProvisioner) upsertSecretMerge(ctx context.Context, name string, d } } + // A fenced write never moves the projection backwards. The stamp on the + // object is what a replica compares against: a delayed write from a replica + // whose view is older must not land after a newer one, which is exactly how + // a removed warehouse's logins come back. + if revision > 0 { + if written := projectionRevisionOf(existing); written > revision { + slog.Debug("trino reconcile: not overwriting a newer projection", + "secret", name, "written", written, "holding", revision) + return nil + } + } + // Merge data: preserve existing keys, overwrite ours. merged := make(map[string][]byte, len(existing.Data)+len(data)) for k, v := range existing.Data { @@ -2738,13 +3168,81 @@ func (p *TrinoProvisioner) upsertSecretMerge(ctx context.Context, name string, d } existing.Labels["app"] = "trino" existing.Labels["duckgres/managed"] = "true" + if revision > 0 { + if existing.Annotations == nil { + existing.Annotations = map[string]string{} + } + existing.Annotations[TrinoProjectionRevisionAnnotation] = strconv.FormatInt(revision, 10) + } + // The resourceVersion carried by `existing` makes this a compare-and-swap: + // a concurrent write between the read and here is a conflict rather than a + // silent last-writer-wins, and the next tick re-reads. if _, err := secrets.Update(ctx, existing, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("update secret %s (merge): %w", name, err) } return nil } +// TrinoProjectionRevisionAnnotation records which accepted projection a +// projected object was written from. +const TrinoProjectionRevisionAnnotation = "duckgres.posthog.com/projection-revision" + +func projectionAnnotations(revision int64) map[string]string { + if revision <= 0 { + return nil + } + return map[string]string{TrinoProjectionRevisionAnnotation: strconv.FormatInt(revision, 10)} +} + +func projectionRevisionOf(object metav1.Object) int64 { + value, present := object.GetAnnotations()[TrinoProjectionRevisionAnnotation] + if !present { + return 0 + } + revision, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0 + } + return revision +} + +// TrinoProjectionFence is the durable record of which authorization projection +// a pooled cell accepts. +// +// Accept builds and records one: the builder runs INSIDE the fence's +// transaction and is handed the source rows it read, so the bytes and the +// revision describe one read. It refuses when this process may not advance the +// projection - it does not hold the pool's authority, or it is not the +// publisher the deployment currently wants. An older control plane publishes +// older authorization rules, and a revision counter cannot notice that: it +// orders acceptances, not rule sets. +// +// Accepted reads the record without changing it. EVERY replica needs it: they +// all serve the bundle and all project the auth Secret, so each must be able to +// ask whether what it holds is the accepted projection. +type TrinoProjectionFence interface { + Accept(ctx context.Context, build func(orgs []configstore.TrinoEnabledOrg) (digest string, err error)) (revision int64, err error) + Accepted(ctx context.Context) (digest string, revision int64, ok bool) +} + +// ErrTrinoProjectionNotAdvanceable means this process may not ADVANCE the +// accepted projection - it is not the authority, or not the desired publisher. +// +// It is not a failure of the reconcile: the replica still builds and serves the +// projection, and the fence decides whether those bytes may be published. Every +// replica but one is in this state at any moment, so treating it as an error +// would mark every pooled warehouse failed on every replica that is not the +// leader. +var ErrTrinoProjectionNotAdvanceable = errors.New("this control plane may not advance the accepted projection") + +// SetProjectionFence installs the durable projection fence for a pooled cell. +func (p *TrinoProvisioner) SetProjectionFence(fence TrinoProjectionFence) { + p.credMu.Lock() + defer p.credMu.Unlock() + p.projectionFence = fence +} + // replaceSecret is the sole-owner Secret writer: the given data map // becomes the Secret's ENTIRE contents, so keys this call doesn't name are // deleted. That is the point for the tenant-password Secret — an org that diff --git a/controlplane/provisioner/trino_provisioner_test.go b/controlplane/provisioner/trino_provisioner_test.go index 4282e832d..f46325030 100644 --- a/controlplane/provisioner/trino_provisioner_test.go +++ b/controlplane/provisioner/trino_provisioner_test.go @@ -2201,3 +2201,87 @@ func TestReconcileOPABundle_ScopeGroupOwnsTheSameCatalog(t *testing.T) { t.Errorf("scope = %#v, want the team's schemas and relations", scope) } } + +// With the Gateway's admission restriction on, a catalog that exists and a +// healthy coordinator are not enough: the Gateway refuses to dispatch work for +// a tenant whose publication has not committed. Reporting Ready then would tell +// an operator the warehouse is queryable while every query it receives is +// refused. +func TestTenantAdmissionGateHoldsAWarehouseAtProvisioning(t *testing.T) { + provisioner := &TrinoProvisioner{} + outcomes := map[string]catalogOutcome{ + "org-admitted": {Existed: true}, + "org-waiting": {Created: true}, + "org-failed": {Err: errors.New("catalog refused")}, + "org-pending": {Pending: true, PendingReason: "waiting for a tenant password"}, + } + provisioner.SetTenantAdmissionGate(func(orgID string) (bool, string) { + return orgID == "org-admitted", "waiting for the pool to admit this warehouse" + }) + + provisioner.applyTenantAdmissionGate(outcomes) + + if outcomes["org-admitted"].Pending || outcomes["org-admitted"].Err != nil { + t.Fatalf("an admitted tenant was held back: %+v", outcomes["org-admitted"]) + } + if !outcomes["org-waiting"].Pending || outcomes["org-waiting"].PendingReason == "" { + t.Fatalf("an unadmitted tenant reads as ready: %+v", outcomes["org-waiting"]) + } + // An existing failure is more specific than "not admitted yet" and must not + // be overwritten by it. + if outcomes["org-failed"].Err == nil { + t.Fatalf("a catalog failure was replaced by the admission gate: %+v", outcomes["org-failed"]) + } + if outcomes["org-pending"].PendingReason != "waiting for a tenant password" { + t.Fatalf("an existing pending reason was overwritten: %+v", outcomes["org-pending"]) + } +} + +// Without the gate - every cell that is not a pooled one with the restriction +// enabled - nothing changes. +func TestTenantAdmissionGateIsInertWhenUnset(t *testing.T) { + provisioner := &TrinoProvisioner{} + outcomes := map[string]catalogOutcome{"org-a": {Existed: true}} + provisioner.applyTenantAdmissionGate(outcomes) + if outcomes["org-a"].Pending { + t.Fatalf("an outcome changed with no gate installed: %+v", outcomes["org-a"]) + } +} + +// A pooled cell has no fixed coordinator, so the legacy readiness probe - which +// asks one coordinator for its node inventory - cannot apply to it. Treating +// that as a failure marked EVERY pooled warehouse failed on every tick with +// "no coordinator client for node inventory"; the pool proves the same thing +// per member, at admission. +func TestPooledCellSkipsTheFixedCoordinatorReadinessProbe(t *testing.T) { + orgs := []configstore.TrinoEnabledOrg{ + {OrgID: "42", DatabaseName: "db42", CellID: testCellID, RootPasswordHash: "$2a$10$h"}, + } + h := newTestTrinoProvisioner(t, orgs, map[string]*configstore.ManagedWarehouse{"42": readyWarehouse("42")}) + h.catalog.nodesErr = ErrTrinoNodeInventoryUnavailable + + if err := h.provisioner.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile with a pooled catalog client: %v", err) + } + + if state := h.store.states["42"]; state.State == configstore.ManagedWarehouseStateFailed { + t.Fatalf("a pooled warehouse was marked failed by a probe that does not apply to it: %+v", state) + } +} + +// The legacy path is unchanged: a fixed cell whose coordinator cannot answer is +// still a failure, because there the inventory IS the readiness evidence. +func TestFixedCellStillFailsWhenTheNodeInventoryIsUnreadable(t *testing.T) { + orgs := []configstore.TrinoEnabledOrg{ + {OrgID: "42", DatabaseName: "db42", CellID: testCellID, RootPasswordHash: "$2a$10$h"}, + } + h := newTestTrinoProvisioner(t, orgs, map[string]*configstore.ManagedWarehouse{"42": readyWarehouse("42")}) + h.catalog.nodesErr = errors.New("coordinator unreachable") + + if err := h.provisioner.Reconcile(context.Background()); err == nil { + t.Fatal("an unreadable node inventory was accepted on a fixed cell") + } + if state := h.store.states["42"]; state.State != configstore.ManagedWarehouseStateFailed { + t.Fatalf("org state = %+v, want Failed", state) + } +} diff --git a/controlplane/trino_fleet.go b/controlplane/trino_fleet.go index d4b92241d..1fadd38b9 100644 --- a/controlplane/trino_fleet.go +++ b/controlplane/trino_fleet.go @@ -103,3 +103,15 @@ func (w *trinoWiring) bundlePath() string { } return "/bundles/trino/" + w.Cell.PublicID } + +// byStoredID finds the wiring of one cell by its stored identity. Callers that +// need a cell's credentials must use this rather than taking the first entry of +// the fleet: a pool must never be certified with another cell's observer. +func (f trinoFleet) byStoredID(storedID string) *trinoWiring { + for _, wire := range f { + if wire.Cell.ID == storedID { + return wire + } + } + return nil +} diff --git a/controlplane/trino_inputs.go b/controlplane/trino_inputs.go index 8ed3e8676..76d54742a 100644 --- a/controlplane/trino_inputs.go +++ b/controlplane/trino_inputs.go @@ -130,6 +130,9 @@ func trinoProvisionerEnabled() bool { // Registered cells share projections across their independently scheduled backends. // Only the legacy cell claims unassigned tenants. type trinoCell struct { + // Mode selects the compute topology. An empty value is the existing fixed + // blue/green cell. + Mode string CatalogManagement string ID string PublicID string @@ -308,6 +311,18 @@ func buildTrinoCellWiring(store trinoWiringStore, kc kubernetes.Interface, duckl // is https but dials the in-cluster Service address (see // envTrinoCoordinatorServerName). catalogClient := provisioner.NewTrinoCatalogHTTPClient(cell.CoordinatorURL, opa.AdminPrincipal, "", cell.TLSServerName) + // A shared-pool cell has no fixed coordinator: its instances come and go, + // and the URL above would be empty. Its catalogs are published directly to + // the shared store under the pool's own authority fence instead, so the + // provisioner's reconcile loop is unchanged while the write path stops + // depending on any one replaceable coordinator. + // + // The writer is attached later, when the pool operator wins its authority + // (SetPoolCatalogWriter). Until then this cell publishes nothing rather + // than publishing through a coordinator that does not exist. + if cell.Mode == trinoPoolModeShared { + catalogClient = newUnavailableTrinoCatalogClient(cell.ID) + } var additional []provisioner.TrinoCatalogClient var managed *provisioner.TrinoManagedCatalogOpts if cell.CatalogManagement != "" { diff --git a/controlplane/trino_pool_binding.go b/controlplane/trino_pool_binding.go new file mode 100644 index 000000000..d9f11385e --- /dev/null +++ b/controlplane/trino_pool_binding.go @@ -0,0 +1,106 @@ +//go:build kubernetes + +package controlplane + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + "strings" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/provisioner" +) + +// The canonical principal/tenant binding. +// +// The Gateway's pooled admission gate restricts dispatch to principals whose +// tenant is admitted. It is NOT a second authenticator: Trino still verifies +// the credential, and OPA still authorizes the query. For that restriction to +// be sound the Gateway's lookup key has to be exactly the principal Trino +// authenticates, and the Gateway cannot derive that itself — it would have to +// guess at username projection and host qualification. +// +// So duckgres publishes the binding authoritatively, derived from the SAME +// projection that writes the coordinator's password.db. If the two ever +// disagree, the gate would either block a legitimate user or admit a principal +// Trino rejects; deriving both from one function is what prevents that. +// +// A warehouse has MANY principals: the bare database name (the org's root +// login) plus one per org user. All of them belong to the same tenant. + +// trinoPoolTenantBinding is one tenant's authoritative principal set. +type trinoPoolTenantBinding struct { + // Tenant is the Gateway's admission key: the org id. + Tenant string `json:"tenant"` + // Catalog is the org's Trino catalog, for operator readability. The gate + // does not key on it. + Catalog string `json:"catalog"` + // Principals are the exact strings the coordinator's password file + // contains, sorted for a stable revision. + Principals []string `json:"principals"` + // Revision changes whenever the principal set changes, so a stale binding + // is detectable rather than silently served. + Revision string `json:"revision"` +} + +// trinoPoolTenantBindingFor derives one org's binding. +// +// Principals that the auth-file projection would REFUSE are dropped here too. +// The projection allowlist exists because duckgres barely validates usernames +// and a `:` or a newline would let whoever can create org users append lines to +// password.db; a username that cannot be projected never reaches password.db, +// so publishing it would bind a principal that can never authenticate. +func trinoPoolTenantBindingFor(org configstore.TrinoEnabledOrg) trinoPoolTenantBinding { + principals := map[string]bool{} + // The bare database name is the org's root login. It is always present: + // ListTrinoEnabledOrgs only returns orgs that have one. + if root := strings.TrimSpace(org.TrinoPrincipal()); root != "" { + principals[root] = true + } + for _, user := range org.Users { + if !provisioner.ProjectableTrinoUsername(user.Username) { + continue + } + principals[org.TrinoUserPrincipal(user.Username)] = true + } + + ordered := make([]string, 0, len(principals)) + for principal := range principals { + ordered = append(ordered, principal) + } + sort.Strings(ordered) + + return trinoPoolTenantBinding{ + Tenant: org.OrgID, + Catalog: configstore.TrinoCatalogName(org.DatabaseName), + Principals: ordered, + Revision: trinoPoolBindingRevision(ordered), + } +} + +// trinoPoolBindingRevision is a stable digest of the principal set. Adding, +// removing or renaming a login changes it, which is what tells the operator a +// tenant's binding has to be republished before the new login can be dispatched. +func trinoPoolBindingRevision(principals []string) string { + digest := sha256.New() + for _, principal := range principals { + _, _ = digest.Write([]byte(principal)) + _, _ = digest.Write([]byte{0}) + } + return hex.EncodeToString(digest.Sum(nil))[:32] +} + +// trinoPoolBindingsFor derives the bindings of every org a pool serves, sorted +// by tenant so a republication decision is reproducible. +func trinoPoolBindingsFor(orgs []configstore.TrinoEnabledOrg, poolID string) []trinoPoolTenantBinding { + bindings := make([]trinoPoolTenantBinding, 0, len(orgs)) + for _, org := range orgs { + if org.CellID != poolID { + continue + } + bindings = append(bindings, trinoPoolTenantBindingFor(org)) + } + sort.Slice(bindings, func(i, j int) bool { return bindings[i].Tenant < bindings[j].Tenant }) + return bindings +} diff --git a/controlplane/trino_pool_binding_test.go b/controlplane/trino_pool_binding_test.go new file mode 100644 index 000000000..97ddf865d --- /dev/null +++ b/controlplane/trino_pool_binding_test.go @@ -0,0 +1,132 @@ +//go:build kubernetes + +package controlplane + +import ( + "strings" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/provisioner" +) + +func bindingOrg(users ...string) configstore.TrinoEnabledOrg { + org := configstore.TrinoEnabledOrg{ + OrgID: "org-a", DatabaseName: "acme", CellID: "registered:cell-001", + RootPasswordHash: "hash", + } + for _, username := range users { + org.Users = append(org.Users, configstore.TrinoOrgUser{Username: username, PasswordHash: "hash"}) + } + return org +} + +// A warehouse has many logins and they all belong to one tenant. A binding that +// carried only the root principal would block every named user at the gate. +func TestBindingCoversEveryLoginOfTheWarehouse(t *testing.T) { + binding := trinoPoolTenantBindingFor(bindingOrg("analyst", "dagster")) + + if binding.Tenant != "org-a" { + t.Fatalf("tenant = %q", binding.Tenant) + } + want := map[string]bool{"acme": true, "acme.analyst": true, "acme.dagster": true} + if len(binding.Principals) != len(want) { + t.Fatalf("principals = %v", binding.Principals) + } + for _, principal := range binding.Principals { + if !want[principal] { + t.Errorf("unexpected principal %q", principal) + } + } +} + +// The binding must contain EXACTLY what the coordinator's password file +// contains. Deriving the two from different rules is how a gate ends up +// blocking a user Trino would authenticate, or admitting one it would not. +func TestBindingMatchesTheProjectedAuthFile(t *testing.T) { + org := bindingOrg("analyst", "dagster") + binding := trinoPoolTenantBindingFor(org) + + passwordDB, _ := provisioner.BuildTrinoAuthFiles([]configstore.TrinoEnabledOrg{org}, + provisioner.TrinoClusterPrincipals{AdminPasswordHash: "admin-hash", ObserverPasswordHash: "observer-hash"}) + + projected := map[string]bool{} + for _, line := range strings.Split(strings.TrimSpace(passwordDB), "\n") { + if line == "" { + continue + } + principal, _, found := strings.Cut(line, ":") + if !found { + continue + } + // The cluster's own operational principals are not tenant principals. + if strings.HasPrefix(principal, "__") { + continue + } + projected[principal] = true + } + + if len(projected) == 0 { + t.Fatal("the auth-file projection produced no tenant principals") + } + for _, principal := range binding.Principals { + if !projected[principal] { + t.Errorf("binding publishes %q, which password.db does not contain", principal) + } + delete(projected, principal) + } + for principal := range projected { + t.Errorf("password.db contains %q, which the binding does not publish", principal) + } +} + +// A username the auth-file projection refuses never reaches password.db, so it +// can never authenticate. Publishing it would advertise a principal Trino +// rejects. +func TestBindingDropsUnprojectableUsernames(t *testing.T) { + binding := trinoPoolTenantBindingFor(bindingOrg("analyst", "bad:user", "with space", "line\nbreak")) + for _, principal := range binding.Principals { + if strings.ContainsAny(principal, ": \n") { + t.Errorf("binding published an unprojectable principal %q", principal) + } + } + if len(binding.Principals) != 2 { + t.Fatalf("principals = %v, want the root login and the one valid user", binding.Principals) + } +} + +// The revision is what tells the operator a tenant's binding must be +// republished before a new login can be dispatched. +func TestBindingRevisionTracksThePrincipalSet(t *testing.T) { + base := trinoPoolTenantBindingFor(bindingOrg("analyst")) + same := trinoPoolTenantBindingFor(bindingOrg("analyst")) + if base.Revision != same.Revision { + t.Fatal("the revision is not stable for an unchanged principal set") + } + // Ordering of the org's users must not change the revision: it is a set. + reordered := trinoPoolTenantBindingFor(bindingOrg("dagster", "analyst")) + ordered := trinoPoolTenantBindingFor(bindingOrg("analyst", "dagster")) + if reordered.Revision != ordered.Revision { + t.Fatal("the revision depends on user ordering") + } + if ordered.Revision == base.Revision { + t.Fatal("adding a login did not change the revision") + } + removed := trinoPoolTenantBindingFor(bindingOrg()) + if removed.Revision == base.Revision { + t.Fatal("removing a login did not change the revision") + } +} + +// A pool publishes bindings only for the orgs it owns. Publishing another +// cell's tenant would admit it on a pool that does not serve it. +func TestBindingsAreScopedToThePool(t *testing.T) { + mine := bindingOrg("analyst") + theirs := bindingOrg("analyst") + theirs.OrgID, theirs.CellID = "org-b", "registered:cell-002" + + bindings := trinoPoolBindingsFor([]configstore.TrinoEnabledOrg{mine, theirs}, "registered:cell-001") + if len(bindings) != 1 || bindings[0].Tenant != "org-a" { + t.Fatalf("bindings = %+v", bindings) + } +} diff --git a/controlplane/trino_pool_catalog.go b/controlplane/trino_pool_catalog.go new file mode 100644 index 000000000..ce40652d0 --- /dev/null +++ b/controlplane/trino_pool_catalog.go @@ -0,0 +1,532 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/provisioner" + "github.com/posthog/duckgres/controlplane/trinocatalog" +) + +// The catalog writer bridge. +// +// Today the provisioner creates a catalog by issuing CREATE CATALOG against a +// coordinator, which then writes the shared catalog table itself. That routes +// provisioning through a replaceable compute instance and inherits the +// asynchronous SQL-DDL cancellation ambiguity: a timed-out statement leaves an +// outcome nobody can resolve. +// +// This adapter presents the fenced direct publisher through the interface the +// provisioner already uses, so the reconcile loop is unchanged and the writes +// become a single fenced transaction with a journal. It is selected per cell +// and only when explicitly enabled; every other cell keeps the coordinator path +// byte for byte. +// +// Node inventory is NOT something the catalog store can answer. It is delegated +// to the existing coordinator client, so readiness still comes from the live +// cluster rather than from a table. +const ( + envTrinoPoolCatalogWriter = "DUCKGRES_TRINO_POOL_CATALOG_WRITER_ENABLED" + envTrinoPoolCatalogBootstrap = "DUCKGRES_TRINO_POOL_CATALOG_BOOTSTRAP" + envTrinoPoolCatalogDSNFile = "DUCKGRES_TRINO_POOL_CATALOG_DSN_FILE" + // envTrinoPoolCatalogSchema names the schema the catalog tables live in. + // The publisher credential carries a database and no schema, and the role's + // privileges are on the cell's schema only. + envTrinoPoolCatalogSchema = "DUCKGRES_TRINO_POOL_CATALOG_SCHEMA" + // envTrinoPoolCatalogCellID names the catalog store's PARTITION - the value + // every coordinator of the cell reads with `catalog-store.cell-id`. + // + // It is REQUIRED with the catalog writer and has no default. The pool id + // carries duckgres's reserved `registered:` prefix and doubles as the + // warehouse-ASSIGNMENT key, so deriving one from the other would force a + // cluster to spell its store partition `registered:` and would re-key + // every coordinator's read the day a pool's logical id changes. Defaulting + // to it would be worse than refusing: the publisher would write rows under + // a partition no coordinator reads, and a store that answers every query + // with an empty catalog set looks exactly like a store nobody has published + // to yet. The chart renders this from the same value it renders the + // coordinators' `catalog-store.cell-id` from. + envTrinoPoolCatalogCellID = "DUCKGRES_TRINO_POOL_CATALOG_CELL_ID" + + // trinoPoolCatalogBootstrapBudget bounds the additive DDL this runs during + // startup wiring. An unreachable database must fail readably rather than + // hang the control plane's boot. + trinoPoolCatalogBootstrapBudget = 30 * time.Second +) + +// trinoPoolSchemaPattern is the unquoted-identifier shape. The schema is +// interpolated into a connection parameter, so anything needing quotes is +// refused rather than escaped. +var trinoPoolSchemaPattern = regexp.MustCompile(`^[a-z_][a-z0-9_]{0,62}$`) + +// trinoPoolCatalogCellIDPattern is the shape the publisher accepts for a store +// partition. It matches what the publisher itself validates, so an unusable +// value is refused when the writer is built rather than at the first +// publication - by then the rows are already somewhere nobody reads. +var trinoPoolCatalogCellIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,255}$`) + +// trinoPoolRevisionStore records the published catalog revision on the pool +// row. It is an interface so the bridge can be tested without a config store. +type trinoPoolRevisionStore interface { + RecordTrinoPoolPublicationRevision(ctx context.Context, lease configstore.TrinoPoolLease, poolID string, revision int64) error +} + +// trinoPoolCatalogWriter adapts the fenced publisher to the provisioner's +// catalog client. +type trinoPoolCatalogWriter struct { + db *sql.DB + // cellID is the catalog store's PARTITION: the `cell_id` column every + // coordinator of this cell filters on. It is a Trino-side identity. + cellID string + // poolID is duckgres's own pool identity, which the pool row is keyed by and + // which the held lease names. The two are deliberately separate values; the + // checkpoint below must address this one, never the partition. + poolID string + // store records the published revision on the pool row, under the same + // authority the publication itself was fenced by. + store trinoPoolRevisionStore + // authority returns the pool lease this control plane currently holds. The + // catalog writer fence is the SAME authority as the operator's: the writer + // epoch is the pool's authority epoch and the writer identity is the + // per-process owner. Building the publisher per call, rather than once at + // startup, is what makes that true - a process that has not won the pool + // cannot write catalogs, and a superseded one stops being able to. + authority func() (configstore.TrinoPoolLease, bool) + // nodes is the live coordinator client. The catalog store knows nothing + // about cluster membership, and inventing an answer here would make the + // provisioner's readiness check meaningless. + nodes provisioner.TrinoCatalogClient +} + +// publisher builds a fenced publisher for the CURRENT authority. It refuses +// when this process holds no lease, so an unelected or superseded replica +// cannot publish at all. +func (w *trinoPoolCatalogWriter) publisher() (*trinocatalog.Publisher, error) { + lease, ok := w.authority() + if !ok || lease.Epoch < 1 { + // Both sentinels matter. ErrNotWriter keeps the publish path treating + // this as a DECISION rather than a lost outcome to resolve, and + // ErrTrinoCatalogNotThisReplica tells the provisioner's reconcile that + // this is not its work - so it leaves the tenants' state rows alone + // instead of marking every pooled org Failed on every non-leader. + return nil, fmt.Errorf("%w: this control plane does not hold the pool authority (%w)", + trinocatalog.ErrNotWriter, provisioner.ErrTrinoCatalogNotThisReplica) + } + return trinocatalog.NewPublisher(w.db, w.cellID, lease.Owner, lease.Epoch) +} + +// ClaimWriter takes the catalog store's writer fence for the lease the operator +// just acquired. It is called once per leadership term, not at startup: the +// claim has to follow the pool authority, or every replica would claim the cell +// on boot and the fence would distinguish nothing. +func (w *trinoPoolCatalogWriter) ClaimWriter(ctx context.Context) error { + publisher, err := w.publisher() + if err != nil { + return err + } + state, err := publisher.Takeover(ctx) + if err != nil { + return fmt.Errorf("claim catalog writer: %w", err) + } + // Checkpoint the watermark from the state the takeover just read. + // + // The catalog store is authoritative for which revision is published; the + // pool row only CACHES it for the admission gate. A previous term can have + // committed a catalog and then failed to record the revision, and nothing + // else republishes it - later catalogs already exist, so no later mutation + // arrives to carry the number forward. Reading it here is the bounded + // recovery: a leadership change is exactly when somebody can fix it. + // + // A failed checkpoint fails the claim. Installing the writer anyway would + // leave the gate believing an older revision, which is how a tenant is + // admitted and reported ready without its catalog. + if err := w.checkpoint(ctx, state.Revision); err != nil { + return err + } + return nil +} + +// PublishedRevision reports the revision the catalog store itself is at. +// +// This is the authority for the admission gate: the pool row's +// publication_revision is a cache of it, and a cache that failed to update is +// indistinguishable from "nothing new was published" unless somebody asks the +// store. +func (w *trinoPoolCatalogWriter) PublishedRevision(ctx context.Context) (int64, error) { + publisher, err := w.publisher() + if err != nil { + return 0, err + } + state, err := publisher.State(ctx) + if err != nil { + return 0, fmt.Errorf("read catalog writer state: %w", err) + } + return state.Revision, nil +} + +// checkpoint records a revision on the pool row under the CURRENT authority. +func (w *trinoPoolCatalogWriter) checkpoint(ctx context.Context, revision int64) error { + lease, held := w.authority() + if !held || w.store == nil || revision <= 0 { + return nil + } + if err := w.store.RecordTrinoPoolPublicationRevision(ctx, lease, w.poolID, revision); err != nil { + return fmt.Errorf("checkpoint published catalog revision %d: %w", revision, err) + } + return nil +} + +func (w *trinoPoolCatalogWriter) ListNodes(ctx context.Context) ([]provisioner.TrinoNode, error) { + if w.nodes == nil { + // A pooled cell has no fixed coordinator to take an inventory from. The + // sentinel is what tells the provisioner's readiness step that this + // probe does not APPLY here, as opposed to failing - the pool proves the + // same thing per member, at admission, against the instance that will + // actually serve the tenant. + return nil, provisioner.ErrTrinoNodeInventoryUnavailable + } + return w.nodes.ListNodes(ctx) +} + +// ListCatalogs reads the published set straight from the store, which is the +// desired state rather than one coordinator's applied view. +func (w *trinoPoolCatalogWriter) ListCatalogs(ctx context.Context) ([]string, error) { + rows, err := w.db.QueryContext(ctx, + `SELECT catalog_name FROM trino_catalogs WHERE cell_id = $1 ORDER BY catalog_name`, w.cellID) + if err != nil { + return nil, fmt.Errorf("list published catalogs: %w", err) + } + defer func() { _ = rows.Close() }() + + var catalogs []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("read published catalog: %w", err) + } + catalogs = append(catalogs, name) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read published catalogs: %w", err) + } + return catalogs, nil +} + +// CatalogConnectors reports which connector each published catalog declares. +// +// The managed-Hoglake path refuses to adopt an existing catalog it cannot +// inspect: an org whose catalog is still DuckLake must be migrated explicitly, +// never silently re-pointed at Hoglake metadata. On a coordinator-mediated cell +// that inspection is a `system.metadata.catalogs` query; a pooled cell has no +// fixed coordinator to ask, so the same question is answered from the store the +// coordinators reconcile FROM. +// +// What this reports is the PUBLISHED definition, not a running coordinator's +// applied state, and the two differ while a member is still catching up. That +// is sound for this check and only this check - it decides whether duckgres may +// replace its own published definition. Whether any member has actually applied +// it stays with pool admission, which proves it per member against the +// published revision; nothing here may be read as evidence that a catalog is +// operational. +func (w *trinoPoolCatalogWriter) CatalogConnectors(ctx context.Context) (map[string]string, error) { + rows, err := w.db.QueryContext(ctx, + `SELECT catalog_name, connector_name FROM trino_catalogs WHERE cell_id = $1`, w.cellID) + if err != nil { + return nil, fmt.Errorf("read published catalog connectors: %w", err) + } + defer func() { _ = rows.Close() }() + + connectors := map[string]string{} + for rows.Next() { + var name, connector string + if err := rows.Scan(&name, &connector); err != nil { + return nil, fmt.Errorf("read published catalog connector: %w", err) + } + if name == "" || connector == "" { + return nil, fmt.Errorf("published catalog inventory is incomplete for cell %s", w.cellID) + } + connectors[name] = connector + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read published catalog connectors: %w", err) + } + return connectors, nil +} + +func (w *trinoPoolCatalogWriter) CreateCatalog(ctx context.Context, name string, properties map[string]string) error { + return w.publish(ctx, trinocatalog.Mutation{ + Operation: trinocatalog.OperationAddOrReplace, + CatalogName: name, + ConnectorName: properties["connector.name"], + Properties: connectorProperties(properties), + }) +} + +// AlterCatalog is the same publication as a create: the store holds one row per +// catalog, and the coordinators reconcile to whatever it says. +func (w *trinoPoolCatalogWriter) AlterCatalog(ctx context.Context, name string, properties map[string]string) error { + return w.CreateCatalog(ctx, name, properties) +} + +func (w *trinoPoolCatalogWriter) DropCatalog(ctx context.Context, name string) error { + return w.publish(ctx, trinocatalog.Mutation{ + Operation: trinocatalog.OperationRemove, + CatalogName: name, + }) +} + +// publish derives the operation id from the intent itself, so a retry of the +// same intent is recognized as a replay and returns the recorded revision +// instead of publishing twice. A CHANGED intent is a different operation, which +// is what advances the revision. +func (w *trinoPoolCatalogWriter) publish(ctx context.Context, mutation trinocatalog.Mutation) error { + // The operation id includes the revision the store is at RIGHT NOW, not + // only the intent's content. + // + // Content alone was wrong in a way that silently lost a catalog: create X, + // drop X, then create X again with identical properties produced the same + // operation id as the first create, hit the journal, and returned the old + // revision without writing anything - so the recreated catalog was never + // republished and no coordinator ever saw it again. Including the current + // revision makes each of those three intents its own operation, while an + // immediate retry of the SAME intent (nothing else committed in between) + // still resolves as a replay. + lease, held := w.authority() + publisher, err := w.publisher() + if err != nil { + return err + } + state, err := publisher.State(ctx) + if err != nil { + return fmt.Errorf("read catalog writer state: %w", err) + } + mutation.OperationID = fmt.Sprintf("catalog.%s.r%d.%s", mutation.CatalogName, state.Revision, mutation.PayloadHash()[:16]) + + result, err := publisher.Apply(ctx, mutation) + switch { + case err == nil: + case isTerminalPublishError(err): + // A fence refusal or a changed intent is a decision, not an unknown + // outcome. Resolving it against the journal would report somebody + // else's row as this call's success. + return err + default: + // Anything else may be a lost COMMIT, whose outcome is UNKNOWN. + // Resolving by operation id alone cannot answer it: the id carries the + // revision this attempt read, and a committed mutation has already + // moved it, so the retry computes a different id and misses. The INTENT + // plus "later than the revision I read" identifies the same commit. + resolved, resolveErr := publisher.ResolveIntentSince(ctx, mutation.CatalogName, mutation.PayloadHash(), state.Revision) + if resolveErr != nil || resolved == nil { + return err + } + result = *resolved + } + + // The published revision is the gate a candidate must have applied before it + // can be admitted. Recording it is what arms that gate; without it every + // coordinator is certified at revision zero and a member missing the newest + // tenant looks current. + // + // A failed checkpoint is RETURNED, not logged and dropped. The catalog is + // committed either way, but nothing republishes it: later catalogs already + // exist, so no future mutation carries the number forward, and the gate + // would keep certifying members against a revision that predates this + // tenant - admitting it, and reporting the warehouse ready, without its + // catalog. Surfacing it holds that org not-ready until a later tick or the + // next leadership claim checkpoints the watermark, and the publication + // itself resolves as a replay. + if held && w.store != nil && result.Revision > 0 { + if err := w.store.RecordTrinoPoolPublicationRevision(ctx, lease, w.poolID, result.Revision); err != nil { + slog.Error("Trino pool publication revision could not be recorded; tenant admission stays closed until it is.", + "pool", w.poolID, "cell", w.cellID, "revision", result.Revision, "error", err) + return fmt.Errorf("checkpoint published catalog revision %d: %w", result.Revision, err) + } + } + return nil +} + +// isTerminalPublishError reports an outcome the publisher DECIDED, as opposed +// to one it never got to observe. +func isTerminalPublishError(err error) bool { + return errors.Is(err, trinocatalog.ErrFenced) || + errors.Is(err, trinocatalog.ErrNotWriter) || + errors.Is(err, trinocatalog.ErrIntentChanged) +} + +// connectorProperties strips the connector name, which the store keeps in its +// own column and which is not part of the property map Trino hashes. +func connectorProperties(properties map[string]string) map[string]string { + filtered := make(map[string]string, len(properties)) + for key, value := range properties { + if key == "connector.name" { + continue + } + filtered[key] = value + } + return filtered +} + +// buildTrinoPoolCatalogWriter constructs the bridge for one pool. It returns +// (nil, nil) when the writer is not enabled, which leaves the existing +// coordinator-mediated path in place. +// +// The DSN is read from a file rather than an environment variable because it +// carries the publisher credential, which infra provisions separately from the +// coordinators' read-only reader role. +// +// poolID is duckgres's own identity for the pool; the catalog store's partition +// is a separate, separately configured identity, resolved and REQUIRED below. +func buildTrinoPoolCatalogWriter(poolID string, store trinoPoolRevisionStore, authority func() (configstore.TrinoPoolLease, bool), nodes provisioner.TrinoCatalogClient) (*trinoPoolCatalogWriter, error) { + enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(envTrinoPoolCatalogWriter))) + if err != nil || !enabled { + return nil, nil + } + path := strings.TrimSpace(os.Getenv(envTrinoPoolCatalogDSNFile)) + if path == "" { + return nil, fmt.Errorf("%s is enabled but %s is unset", envTrinoPoolCatalogWriter, envTrinoPoolCatalogDSNFile) + } + raw, err := readRolloutSecretFile(path, 8192) + if err != nil { + return nil, fmt.Errorf("read catalog writer credential: %w", err) + } + // The schema the catalog tables live in. + // + // The credential names a DATABASE and nothing else, and the publisher role + // holds privileges on the cell's schema alone - so unqualified SQL resolves + // against `public`, where that role can neither create nor read anything. + // It is required rather than defaulted: guessing a schema would produce + // exactly that failure at the first publication, on a path an operator has + // no reason to suspect. + schema, err := trinoPoolCatalogSchema() + if err != nil { + return nil, err + } + // The store partition, which is a Trino-side identity rather than this + // pool's. Resolved before the database is opened so an unusable value is a + // startup error rather than a first-publication one. + cellID, err := trinoPoolCatalogCellID() + if err != nil { + return nil, err + } + dsn, err := withSearchPath(strings.TrimSpace(string(raw)), schema) + if err != nil { + return nil, err + } + // pgx, not "postgres": lib/pq is not linked into the control-plane binary, + // so sql.Open("postgres", ...) fails at startup with `unknown driver`. The + // rest of the control plane registers pgx (see storage_meter.go). + db, err := sql.Open("pgx", dsn) + if err != nil { + return nil, fmt.Errorf("open catalog store: %w", err) + } + // One publisher, a handful of connections: every mutation serializes on the + // cell's writer row anyway. + db.SetMaxOpenConns(4) + + writer := &trinoPoolCatalogWriter{db: db, cellID: cellID, poolID: poolID, store: store, authority: authority, nodes: nodes} + + if bootstrap, _ := strconv.ParseBool(strings.TrimSpace(os.Getenv(envTrinoPoolCatalogBootstrap))); bootstrap { + // Somebody has to create the additive tables, because a managed-reader + // coordinator runs no DDL at all. It is explicit so a deployment that + // has not split its database grants yet cannot do it by accident. This + // is additive DDL only and takes no fence, because it publishes nothing. + // + // It is BOUNDED: this runs during startup wiring, so an unreachable + // database must fail with a readable error rather than hang the control + // plane's boot indefinitely. The error keeps its cause so the next + // attempt is diagnosable. + ctx, cancel := context.WithTimeout(context.Background(), trinoPoolCatalogBootstrapBudget) + defer cancel() + bootstrapper, err := trinocatalog.NewPublisher(db, cellID, "duckgres-bootstrap", 1) + if err != nil { + _ = db.Close() + return nil, fmt.Errorf("configure catalog bootstrap: %w", err) + } + if err := bootstrapper.EnsureSchema(ctx); err != nil { + _ = db.Close() + return nil, fmt.Errorf("bootstrap catalog store in schema %q: %w", schema, err) + } + } + return writer, nil +} + +// trinoPoolCatalogSchema resolves and validates the schema the catalog tables +// live in. +// +// The value is interpolated into a connection parameter, so it is checked +// against the unquoted-identifier shape rather than escaped: a schema name that +// needs quoting is a deployment mistake worth refusing, and accepting one here +// would put caller-shaped text into a connection string. +func trinoPoolCatalogSchema() (string, error) { + schema := strings.TrimSpace(os.Getenv(envTrinoPoolCatalogSchema)) + if schema == "" { + return "", fmt.Errorf("%s is enabled but %s is unset: the publisher credential names a database only, and the role's privileges are on the cell's schema", + envTrinoPoolCatalogWriter, envTrinoPoolCatalogSchema) + } + if !trinoPoolSchemaPattern.MatchString(schema) { + return "", fmt.Errorf("%s=%q is not a plain lower-case identifier", envTrinoPoolCatalogSchema, schema) + } + return schema, nil +} + +// trinoPoolCatalogCellID resolves the catalog store's partition. +// +// The value must be the one the cell's coordinators carry in +// `catalog-store.cell-id`: rows written under any other partition are invisible +// to every one of them. It is REQUIRED and fails closed - unset, blank and +// unusable are one answer, because each of them means nobody can say which +// partition this publisher would write to. Defaulting to the pool id would let +// a missing setting publish a full catalog set somewhere no coordinator looks, +// which is indistinguishable from a store nothing has published to yet. +func trinoPoolCatalogCellID() (string, error) { + cellID := strings.TrimSpace(os.Getenv(envTrinoPoolCatalogCellID)) + if cellID == "" { + return "", fmt.Errorf("%s is enabled but %s is unset: it is the partition the cell's coordinators read with catalog-store.cell-id, and there is no safe value to assume", + envTrinoPoolCatalogWriter, envTrinoPoolCatalogCellID) + } + if !trinoPoolCatalogCellIDPattern.MatchString(cellID) { + return "", fmt.Errorf("%s=%q is not a usable catalog store partition", envTrinoPoolCatalogCellID, cellID) + } + return cellID, nil +} + +// withSearchPath pins the connection's search_path to that one schema. +// +// Every statement the publisher issues is unqualified, and the reader side of +// the same tables resolves them the same way, so the schema belongs on the +// connection rather than being threaded through each statement. A DSN that +// already sets a search_path is refused instead of silently overridden: two +// sources for the same setting is how a publisher ends up writing where nobody +// is looking. +func withSearchPath(dsn, schema string) (string, error) { + parsed, err := url.Parse(dsn) + if err != nil { + // A keyword/value DSN ("host=... dbname=...") is not a URL. Rather than + // re-implement that grammar, refuse it: infra provisions a URL. + return "", fmt.Errorf("catalog writer credential is not a postgres:// URL: %w", err) + } + if parsed.Scheme != "postgres" && parsed.Scheme != "postgresql" { + return "", fmt.Errorf("catalog writer credential is not a postgres:// URL") + } + query := parsed.Query() + if existing := strings.TrimSpace(query.Get("search_path")); existing != "" && existing != schema { + return "", fmt.Errorf("catalog writer credential already pins search_path=%q, which disagrees with %s=%q", + existing, envTrinoPoolCatalogSchema, schema) + } + query.Set("search_path", schema) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} diff --git a/controlplane/trino_pool_catalog_hoglake_postgres_test.go b/controlplane/trino_pool_catalog_hoglake_postgres_test.go new file mode 100644 index 000000000..8f5db8c9f --- /dev/null +++ b/controlplane/trino_pool_catalog_hoglake_postgres_test.go @@ -0,0 +1,188 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" +) + +// testCatalogPartition is the catalog store's partition these suites publish +// under. It is deliberately NOT derived from any pool id: the two identities are +// configured separately, and a test that spelled them the same could not see a +// writer that confused one for the other. +const testCatalogPartition = "example-pool-store" + +// scopedCatalogWriter builds the production writer against a real PostgreSQL, +// as the scoped publisher role infra provisions. +func scopedCatalogWriter(t *testing.T, poolID string, store trinoPoolRevisionStore, lease configstore.TrinoPoolLease) *trinoPoolCatalogWriter { + t.Helper() + adminDSN := adminPostgresURL(t) + admin, err := sql.Open("pgx", adminDSN) + if err != nil { + t.Fatalf("open postgres: %v", err) + } + t.Cleanup(func() { _ = admin.Close() }) + + suffix := randomSuffix(t) + schema := "trino_cell_" + suffix + role := "dgpub_" + suffix + password := "pw_" + randomSuffix(t) + + mustExec(t, admin, fmt.Sprintf(`CREATE SCHEMA %s`, schema)) + mustExec(t, admin, fmt.Sprintf(`CREATE ROLE %s LOGIN PASSWORD '%s'`, role, password)) + t.Cleanup(func() { + _, _ = admin.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, schema)) + _, _ = admin.Exec(fmt.Sprintf(`REASSIGN OWNED BY %s TO CURRENT_USER`, role)) + _, _ = admin.Exec(fmt.Sprintf(`DROP OWNED BY %s`, role)) + _, _ = admin.Exec(fmt.Sprintf(`DROP ROLE IF EXISTS %s`, role)) + }) + mustExec(t, admin, fmt.Sprintf(`GRANT USAGE, CREATE ON SCHEMA %s TO %s`, schema, role)) + mustExec(t, admin, fmt.Sprintf(`REVOKE ALL ON SCHEMA public FROM %s`, role)) + + dsnFile := filepath.Join(t.TempDir(), "publisher.dsn") + if err := os.WriteFile(dsnFile, []byte(scopedDSN(t, adminDSN, role, password)), 0o600); err != nil { + t.Fatalf("write dsn file: %v", err) + } + t.Setenv(envTrinoPoolCatalogWriter, "true") + t.Setenv(envTrinoPoolCatalogBootstrap, "true") + t.Setenv(envTrinoPoolCatalogDSNFile, dsnFile) + t.Setenv(envTrinoPoolCatalogSchema, schema) + // The partition is required and has no default, so every suite states it. + t.Setenv(envTrinoPoolCatalogCellID, testCatalogPartition) + + writer, err := buildTrinoPoolCatalogWriter(poolID, store, + func() (configstore.TrinoPoolLease, bool) { return lease, true }, nil) + if err != nil || writer == nil { + t.Fatalf("build the catalog writer: %v", err) + } + t.Cleanup(func() { _ = writer.db.Close() }) + return writer +} + +// recordingRevisionStore is the pool row's checkpoint, with the failure the +// watermark recovery exists for. +type recordingRevisionStore struct { + revision int64 + refuse bool +} + +func (s *recordingRevisionStore) RecordTrinoPoolPublicationRevision(_ context.Context, _ configstore.TrinoPoolLease, _ string, revision int64) error { + if s.refuse { + return errors.New("checkpoint refused") + } + s.revision = revision + return nil +} + +// Managed Hoglake refuses to adopt a catalog whose connector it cannot inspect, +// because silently re-pointing an existing DuckLake catalog at Hoglake metadata +// would be a migration nobody asked for. On a coordinator-mediated cell that +// inspection is a `system.metadata.catalogs` query; a pooled cell has no fixed +// coordinator, so the same question is answered from the store the coordinators +// reconcile from. +func TestPooledCatalogWriterReportsPublishedConnectors(t *testing.T) { + lease := configstore.TrinoPoolLease{PoolID: "registered:cell-001", Owner: "cp-test", Epoch: 1} + writer := scopedCatalogWriter(t, lease.PoolID, &recordingRevisionStore{}, lease) + ctx := context.Background() + if err := writer.ClaimWriter(ctx); err != nil { + t.Fatalf("claim the writer fence: %v", err) + } + + // Nothing published yet: an empty inventory, not an error. A first Hoglake + // catalog has no existing connector to disagree with. + connectors, err := writer.CatalogConnectors(ctx) + if err != nil { + t.Fatalf("read connectors on an empty cell: %v", err) + } + if len(connectors) != 0 { + t.Fatalf("connectors = %v, want none before anything is published", connectors) + } + + if err := writer.CreateCatalog(ctx, "org_legacy", map[string]string{ + "connector.name": "ducklake", "ducklake.data-path": "s3://bucket/legacy/", + }); err != nil { + t.Fatalf("publish the DuckLake catalog: %v", err) + } + if err := writer.CreateCatalog(ctx, "org_new", map[string]string{ + "connector.name": "hoglake", "hoglake.catalog": "org_new", + }); err != nil { + t.Fatalf("publish the Hoglake catalog: %v", err) + } + + connectors, err = writer.CatalogConnectors(ctx) + if err != nil { + t.Fatalf("read connectors: %v", err) + } + // The mismatched one is reported as what it IS, which is what makes the + // adoption check refuse it, and the Hoglake one as hoglake, which is what + // lets an already-published tenant reconcile without being recreated. + if connectors["org_legacy"] != "ducklake" { + t.Fatalf("org_legacy connector = %q, want the existing DuckLake connector to be visible", + connectors["org_legacy"]) + } + if connectors["org_new"] != "hoglake" { + t.Fatalf("org_new connector = %q, want hoglake", connectors["org_new"]) + } +} + +// The watermark the admission gate certifies against comes from the catalog +// store, so a checkpoint that failed after a committed catalog is recoverable. +// +// Without this the number is only ever written as a side effect of a +// publication, and the catalog that failed to record its revision is never +// published again - it already exists. The gate would keep certifying members +// against a revision that predates the tenant. +func TestPooledCatalogWriterRecoversTheWatermarkFromTheStore(t *testing.T) { + lease := configstore.TrinoPoolLease{PoolID: "registered:cell-001", Owner: "cp-test", Epoch: 1} + store := &recordingRevisionStore{} + writer := scopedCatalogWriter(t, lease.PoolID, store, lease) + ctx := context.Background() + if err := writer.ClaimWriter(ctx); err != nil { + t.Fatalf("claim the writer fence: %v", err) + } + + // The catalog commits; the checkpoint of its revision does not. + store.refuse = true + err := writer.CreateCatalog(ctx, "org_acme", map[string]string{ + "connector.name": "hoglake", "hoglake.catalog": "org_acme", + }) + if err == nil { + t.Fatal("a failed revision checkpoint was reported as success") + } + if store.revision != 0 { + t.Fatalf("checkpointed revision = %d, want none recorded", store.revision) + } + + // The catalog IS published - the publication itself committed. + names, err := writer.ListCatalogs(ctx) + if err != nil || len(names) != 1 || names[0] != "org_acme" { + t.Fatalf("published catalogs = %v (err %v), want the committed catalog", names, err) + } + + // Nothing will republish it, so the number has to be recoverable from the + // store's own writer state. That is what the admission gate reads. + published, err := writer.PublishedRevision(ctx) + if err != nil { + t.Fatalf("read the published revision: %v", err) + } + if published < 1 { + t.Fatalf("published revision = %d, want the committed catalog's revision", published) + } + + // And a later claim checkpoints it without any new catalog mutation. + store.refuse = false + if err := writer.ClaimWriter(ctx); err != nil { + t.Fatalf("re-claim the writer fence: %v", err) + } + if store.revision != published { + t.Fatalf("checkpointed revision = %d, want %d recovered at the claim", store.revision, published) + } +} diff --git a/controlplane/trino_pool_catalog_identity_postgres_test.go b/controlplane/trino_pool_catalog_identity_postgres_test.go new file mode 100644 index 000000000..d9e3902ff --- /dev/null +++ b/controlplane/trino_pool_catalog_identity_postgres_test.go @@ -0,0 +1,126 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" +) + +// assertingRevisionStore is the pool row's checkpoint with the real store's +// precondition: RecordTrinoPoolPublicationRevision refuses a pool id that is not +// the one the held lease names. The catalog store's partition must therefore +// never be handed to it, however the deployment spells that partition. +type assertingRevisionStore struct { + t *testing.T + poolID string + revision int64 +} + +func (s *assertingRevisionStore) RecordTrinoPoolPublicationRevision(_ context.Context, lease configstore.TrinoPoolLease, poolID string, revision int64) error { + s.t.Helper() + if poolID != lease.PoolID { + s.t.Fatalf("checkpoint addressed pool %q, the held lease is for %q", poolID, lease.PoolID) + } + s.poolID, s.revision = poolID, revision + return nil +} + +// The catalog store's partition is a TRINO-side identity: it is the value each +// coordinator of the cell reads with `catalog-store.cell-id`, and the rows the +// publisher writes must carry exactly it or every coordinator reconciles an +// empty catalog set. +// +// It is deliberately not derived from the pool's own id. That id carries +// duckgres's reserved `registered:` prefix and doubles as the warehouse +// ASSIGNMENT key, so deriving one from the other would force a cluster to spell +// its store partition `registered:` and would silently re-key every +// coordinator's read the day a pool's logical id changes. The two identities are +// configured separately and this asserts they stay separate: the rows land under +// the configured partition while the pool row's checkpoint still addresses the +// pool. +func TestCatalogWriterPublishesUnderTheConfiguredStorePartition(t *testing.T) { + lease := configstore.TrinoPoolLease{PoolID: "registered:example-pool", Owner: "cp-test", Epoch: 1} + store := &assertingRevisionStore{t: t} + // The helper states the partition, which is a different value from the pool + // id this writer is built for. + writer := scopedCatalogWriter(t, lease.PoolID, store, lease) + + ctx := context.Background() + if err := writer.ClaimWriter(ctx); err != nil { + t.Fatalf("claim the writer fence: %v", err) + } + if err := writer.CreateCatalog(ctx, "org_acme", map[string]string{ + "connector.name": "ducklake", + "ducklake.data-path": "s3://bucket/prefix/", + }); err != nil { + t.Fatalf("publish a catalog: %v", err) + } + + // The definition and the writer state both belong to the partition the + // coordinators read, not to the pool's id. + var cellID string + if err := writer.db.QueryRow(`SELECT cell_id FROM trino_catalogs WHERE catalog_name = 'org_acme'`).Scan(&cellID); err != nil { + t.Fatalf("read the published catalog: %v", err) + } + if cellID != testCatalogPartition { + t.Fatalf("catalog published under cell_id %q, the coordinators read %q", cellID, testCatalogPartition) + } + if err := writer.db.QueryRow(`SELECT cell_id FROM trino_catalog_writer_state`).Scan(&cellID); err != nil { + t.Fatalf("read the writer state: %v", err) + } + if cellID != testCatalogPartition { + t.Fatalf("writer state recorded under cell_id %q, want %q", cellID, testCatalogPartition) + } + + // And the checkpoint reached the pool row, which is a different identity. + if store.poolID != lease.PoolID || store.revision < 1 { + t.Fatalf("checkpoint recorded pool %q at revision %d", store.poolID, store.revision) + } +} + +// The partition is REQUIRED and fails closed, and the three ways it can be +// absent are one answer: each means nobody can say which partition the +// publisher would write to. There is deliberately no fallback to the pool id - +// that would let a missing setting publish a full catalog set under a partition +// no coordinator reads, which looks exactly like a store nothing has been +// published to yet. +func TestCatalogWriterRequiresAStorePartition(t *testing.T) { + dsnFile := filepath.Join(t.TempDir(), "publisher.dsn") + if err := os.WriteFile(dsnFile, []byte("postgres://user:pw@example.invalid:5432/duckgres"), 0o600); err != nil { + t.Fatalf("write dsn file: %v", err) + } + + for name, partition := range map[string]string{ + "unset": "", + "blank": " ", + "unusable": "not a cell id", + } { + t.Run(name, func(t *testing.T) { + t.Setenv(envTrinoPoolCatalogWriter, "true") + t.Setenv(envTrinoPoolCatalogBootstrap, "false") + t.Setenv(envTrinoPoolCatalogDSNFile, dsnFile) + t.Setenv(envTrinoPoolCatalogSchema, "trino_pool") + t.Setenv(envTrinoPoolCatalogCellID, partition) + + writer, err := buildTrinoPoolCatalogWriter("registered:example-pool", nil, + func() (configstore.TrinoPoolLease, bool) { return configstore.TrinoPoolLease{}, false }, nil) + if err == nil { + t.Fatal("the writer was built with no usable store partition") + } + if writer != nil { + t.Fatal("a writer was returned alongside the refusal") + } + // The refusal names the setting, because the operator's next move is + // to render it from whatever the coordinators already carry. + if !strings.Contains(err.Error(), envTrinoPoolCatalogCellID) { + t.Fatalf("the refusal does not name %s: %v", envTrinoPoolCatalogCellID, err) + } + }) + } +} diff --git a/controlplane/trino_pool_catalog_schema_postgres_test.go b/controlplane/trino_pool_catalog_schema_postgres_test.go new file mode 100644 index 000000000..4e2aa492e --- /dev/null +++ b/controlplane/trino_pool_catalog_schema_postgres_test.go @@ -0,0 +1,236 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" +) + +// The publisher's real credential against its real grants. +// +// Infra provisions a role with privileges on the cell's schema ONLY, and its +// connection URL names a database with no schema at all - so every unqualified +// statement the publisher issues resolves against `public`, where that role can +// neither create nor read. A test that uses an administrator DSN with a +// preconfigured search_path cannot see that: it has rights everywhere. This one +// builds the writer through the production builder, as a role that is scoped +// exactly as the real one is. +func TestCatalogWriterPublishesInsideItsGrantedSchema(t *testing.T) { + adminDSN := adminPostgresURL(t) + admin, err := sql.Open("pgx", adminDSN) + if err != nil { + t.Fatalf("open postgres: %v", err) + } + defer func() { _ = admin.Close() }() + + suffix := randomSuffix(t) + schema := "trino_cell_" + suffix + role := "dgpub_" + suffix + password := "pw_" + randomSuffix(t) + + mustExec(t, admin, fmt.Sprintf(`CREATE SCHEMA %s`, schema)) + mustExec(t, admin, fmt.Sprintf(`CREATE ROLE %s LOGIN PASSWORD '%s'`, role, password)) + t.Cleanup(func() { + _, _ = admin.Exec(fmt.Sprintf(`DROP SCHEMA IF EXISTS %s CASCADE`, schema)) + _, _ = admin.Exec(fmt.Sprintf(`REASSIGN OWNED BY %s TO CURRENT_USER`, role)) + _, _ = admin.Exec(fmt.Sprintf(`DROP OWNED BY %s`, role)) + _, _ = admin.Exec(fmt.Sprintf(`DROP ROLE IF EXISTS %s`, role)) + }) + // Exactly the shape infra grants: the cell's schema, and nothing in public. + mustExec(t, admin, fmt.Sprintf(`GRANT USAGE, CREATE ON SCHEMA %s TO %s`, schema, role)) + mustExec(t, admin, fmt.Sprintf(`REVOKE ALL ON SCHEMA public FROM %s`, role)) + + dsnFile := filepath.Join(t.TempDir(), "publisher.dsn") + if err := os.WriteFile(dsnFile, []byte(scopedDSN(t, adminDSN, role, password)), 0o600); err != nil { + t.Fatalf("write dsn file: %v", err) + } + + t.Setenv(envTrinoPoolCatalogWriter, "true") + t.Setenv(envTrinoPoolCatalogBootstrap, "true") + t.Setenv(envTrinoPoolCatalogDSNFile, dsnFile) + t.Setenv(envTrinoPoolCatalogSchema, schema) + t.Setenv(envTrinoPoolCatalogCellID, testCatalogPartition) + + lease := configstore.TrinoPoolLease{PoolID: "registered:example-pool", Owner: "cp-test", Epoch: 1} + writer, err := buildTrinoPoolCatalogWriter(lease.PoolID, nil, + func() (configstore.TrinoPoolLease, bool) { return lease, true }, nil) + if err != nil { + t.Fatalf("build the catalog writer as the scoped publisher role: %v", err) + } + if writer == nil { + t.Fatal("the writer was not built") + } + t.Cleanup(func() { _ = writer.db.Close() }) + + ctx := context.Background() + if err := writer.ClaimWriter(ctx); err != nil { + t.Fatalf("claim the writer fence: %v", err) + } + if err := writer.CreateCatalog(ctx, "org_acme", map[string]string{ + "connector.name": "ducklake", + "ducklake.data-path": "s3://bucket/prefix/", + }); err != nil { + t.Fatalf("publish a catalog as the scoped publisher role: %v", err) + } + + // The rows are in the GRANTED schema, which is where the coordinators read. + var catalogs int + if err := admin.QueryRow( + fmt.Sprintf(`SELECT count(*) FROM %s.trino_catalogs WHERE catalog_name = 'org_acme'`, schema), + ).Scan(&catalogs); err != nil { + t.Fatalf("read the published catalog from %s: %v", schema, err) + } + if catalogs != 1 { + t.Fatalf("%d catalogs in %s, want the published one", catalogs, schema) + } + // And nothing was created in public, which the role cannot write anyway - + // a writer that fell back there would be publishing where nobody reads. + var inPublic int + if err := admin.QueryRow( + `SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'trino_catalogs'`, + ).Scan(&inPublic); err != nil { + t.Fatalf("inspect public: %v", err) + } + if inPublic != 0 { + t.Fatal("the publisher created its tables in public") + } + + // The reader side resolves the same way. + names, err := writer.ListCatalogs(ctx) + if err != nil { + t.Fatalf("list published catalogs: %v", err) + } + if len(names) != 1 || names[0] != "org_acme" { + t.Fatalf("published catalogs = %v", names) + } +} + +// A schema that is not configured at all is refused at build time rather than +// at the first publication, where it would surface as a permission error on a +// path an operator has no reason to suspect. +func TestCatalogWriterRefusesAnUnconfiguredSchema(t *testing.T) { + dsnFile := filepath.Join(t.TempDir(), "publisher.dsn") + if err := os.WriteFile(dsnFile, []byte("postgres://user:pw@example.invalid:5432/duckgres"), 0o600); err != nil { + t.Fatalf("write dsn file: %v", err) + } + t.Setenv(envTrinoPoolCatalogWriter, "true") + t.Setenv(envTrinoPoolCatalogBootstrap, "false") + t.Setenv(envTrinoPoolCatalogDSNFile, dsnFile) + t.Setenv(envTrinoPoolCatalogCellID, testCatalogPartition) + t.Setenv(envTrinoPoolCatalogSchema, "") + + if _, err := buildTrinoPoolCatalogWriter("registered:example-pool", nil, + func() (configstore.TrinoPoolLease, bool) { return configstore.TrinoPoolLease{}, false }, nil); err == nil { + t.Fatal("the writer was built with no schema configured") + } + + // And a name that would need quoting is refused rather than escaped: it is + // interpolated into a connection parameter. + t.Setenv(envTrinoPoolCatalogSchema, `weird"; DROP TABLE x --`) + if _, err := buildTrinoPoolCatalogWriter("registered:example-pool", nil, + func() (configstore.TrinoPoolLease, bool) { return configstore.TrinoPoolLease{}, false }, nil); err == nil { + t.Fatal("a schema name needing quotes was accepted") + } +} + +// adminPostgresURL is the administrative connection these tests create the +// scoped role from. It follows the same convention as the other real-PostgreSQL +// suites: the shared instance unless DUCKGRES_TEST_PG_DSN points elsewhere. +func adminPostgresURL(t *testing.T) string { + t.Helper() + if dsn := strings.TrimSpace(os.Getenv("DUCKGRES_TEST_PG_DSN")); dsn != "" { + // The suites that share this variable accept the keyword form, which + // the writer under test deliberately refuses (infra provisions a URL). + // Convert rather than skip: this test needs a real scoped role, and + // skipping it where the only PostgreSQL is configured that way would + // mean it never runs at all. + return postgresURLFromDSN(t, dsn) + } + const shared = "postgres://postgres:postgres@127.0.0.1:35432/testdb?sslmode=disable" + db, err := sql.Open("pgx", shared) + if err != nil { + t.Skipf("no administrative PostgreSQL available: %v", err) + } + defer func() { _ = db.Close() }() + if err := db.Ping(); err != nil { + t.Skipf("no administrative PostgreSQL available: %v", err) + } + return shared +} + +// postgresURLFromDSN accepts either form and returns a URL. +func postgresURLFromDSN(t *testing.T, dsn string) string { + t.Helper() + if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") { + return dsn + } + settings := map[string]string{} + for _, field := range strings.Fields(dsn) { + key, value, found := strings.Cut(field, "=") + if !found { + t.Skipf("DUCKGRES_TEST_PG_DSN is not a URL or a keyword DSN: %q", dsn) + } + settings[key] = value + } + host, database := settings["host"], settings["dbname"] + if host == "" || database == "" { + t.Skipf("DUCKGRES_TEST_PG_DSN names no host or database: %q", dsn) + } + user := url.User(settings["user"]) + if password, present := settings["password"]; present { + user = url.UserPassword(settings["user"], password) + } + port := settings["port"] + if port == "" { + port = "5432" + } + query := url.Values{} + if sslmode, present := settings["sslmode"]; present { + query.Set("sslmode", sslmode) + } + built := url.URL{Scheme: "postgres", User: user, Host: host + ":" + port, Path: "/" + database, RawQuery: query.Encode()} + return built.String() +} + +func mustExec(t *testing.T, db *sql.DB, statement string) { + t.Helper() + if _, err := db.Exec(statement); err != nil { + t.Fatalf("%s: %v", statement, err) + } +} + +func randomSuffix(t *testing.T) string { + t.Helper() + buffer := make([]byte, 6) + if _, err := rand.Read(buffer); err != nil { + t.Fatalf("random: %v", err) + } + return hex.EncodeToString(buffer) +} + +// scopedDSN rewrites the admin URL to authenticate as the scoped role, keeping +// the host and database. It deliberately carries NO search_path: that is the +// shape infra provisions, and pinning it is the writer's job. +func scopedDSN(t *testing.T, adminDSN, role, password string) string { + t.Helper() + parsed, err := url.Parse(adminDSN) + if err != nil { + t.Skipf("DUCKGRES_TEST_PG_DSN is not a URL (%v); this test needs one", err) + } + parsed.User = url.UserPassword(role, password) + query := parsed.Query() + query.Del("search_path") + parsed.RawQuery = query.Encode() + return parsed.String() +} diff --git a/controlplane/trino_pool_catalog_unavailable.go b/controlplane/trino_pool_catalog_unavailable.go new file mode 100644 index 000000000..875ceac38 --- /dev/null +++ b/controlplane/trino_pool_catalog_unavailable.go @@ -0,0 +1,65 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "fmt" + + "github.com/posthog/duckgres/controlplane/provisioner" +) + +// unavailableTrinoCatalogClient is what a shared-pool cell publishes through +// before its operator has won the pool authority. +// +// It refuses every call rather than silently succeeding. A no-op would be worse +// than an error: the provisioner would record the org as reconciled while +// nothing was published, and the next coordinator to start would serve a +// catalog set missing that tenant. Refusing keeps the org visibly not-ready +// until a control plane actually holds the fence. +type unavailableTrinoCatalogClient struct{ cellID string } + +func newUnavailableTrinoCatalogClient(cellID string) provisioner.TrinoCatalogClient { + return unavailableTrinoCatalogClient{cellID: cellID} +} + +// err wraps the provisioner's "not this replica" sentinel. The distinction is +// load-bearing: a refusal here means this control plane does not own the write +// path, NOT that the warehouse is broken, and the reconcile must leave every +// org's Trino state row alone rather than marking the whole pool Failed on every +// replica that is not the leader. +func (c unavailableTrinoCatalogClient) err() error { + return fmt.Errorf("shared Trino pool %s has no catalog writer on this control plane: %w", + c.cellID, provisioner.ErrTrinoCatalogNotThisReplica) +} + +func (c unavailableTrinoCatalogClient) ListCatalogs(context.Context) ([]string, error) { + return nil, c.err() +} + +func (c unavailableTrinoCatalogClient) ListNodes(context.Context) ([]provisioner.TrinoNode, error) { + return nil, c.err() +} + +func (c unavailableTrinoCatalogClient) CreateCatalog(context.Context, string, map[string]string) error { + return c.err() +} + +func (c unavailableTrinoCatalogClient) AlterCatalog(context.Context, string, map[string]string) error { + return c.err() +} + +func (c unavailableTrinoCatalogClient) DropCatalog(context.Context, string) error { + return c.err() +} + +// CatalogConnectors refuses with the same sentinel rather than being absent. +// +// Managed Hoglake asks the catalog client to inspect an existing catalog's +// connector before adopting it. A client without this method is reported as +// "cannot verify the Hoglake connector", which reads as a broken cell; the +// sentinel says the truthful thing instead - this control plane does not own +// the write path - so the reconcile leaves the org's state row alone. +func (c unavailableTrinoCatalogClient) CatalogConnectors(context.Context) (map[string]string, error) { + return nil, c.err() +} diff --git a/controlplane/trino_pool_config.go b/controlplane/trino_pool_config.go new file mode 100644 index 000000000..2fd7e6638 --- /dev/null +++ b/controlplane/trino_pool_config.go @@ -0,0 +1,256 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// Shared-pool configuration, resolved from the SAME registry file the existing +// fixed blue/green cells come from. A cell opts in with `mode: "shared-pool"` +// plus a `pool` block; everything without it keeps today's behavior byte for +// byte. +// +// The pool deliberately inherits the cell's LOGICAL ASSIGNMENT identity — its +// pool id, its routing group and its namespace — because replacing compute must +// not rewrite which warehouse lives where. The catalog store's `cell_id` is NOT +// one of them: it is a Trino-side partition, configured on its own +// (`DUCKGRES_TRINO_POOL_CATALOG_CELL_ID`, see trino_pool_catalog.go) to match +// what the cell's coordinators read. +const ( + envTrinoPoolEnabled = "DUCKGRES_TRINO_POOL_ENABLED" + envTrinoPoolOperatorEnabled = "DUCKGRES_TRINO_POOL_OPERATOR_ENABLED" + envTrinoPoolGatewayURL = "DUCKGRES_TRINO_POOL_GATEWAY_URL" + + trinoPoolModeFixed = "fixed" + trinoPoolModeShared = "shared-pool" + + maxBlueprintFileBytes = 1 << 20 +) + +// trinoRegisteredPool is the `pool` block of a registered cell. +type trinoRegisteredPool struct { + DesiredInstances int `json:"desired_instances"` + MinServing int `json:"min_serving"` + MaxSurge int `json:"max_surge"` + MaxRepair int `json:"max_repair"` + BlueprintFile string `json:"blueprint_file"` + CoordinatorServicePort int32 `json:"coordinator_service_port"` + NodeEnvironment string `json:"node_environment"` + // TenantAdmission turns on the Gateway's pooled admission restriction for + // this pool. It is off by default because the restriction is deny-only: with + // it on, a tenant whose principals have not been published yet cannot + // dispatch work, which is correct but must be a deliberate choice. + TenantAdmission bool `json:"tenant_admission,omitempty"` +} + +// trinoPoolConfig is one resolved shared pool. +type trinoPoolConfig struct { + PoolID string + PublicID string + RoutingGroup string + Namespace string + Spec configstore.TrinoPoolSpec + Blueprint *trinopool.Blueprint + Pool trinoRegisteredPool + // PublisherImage is the image this deployment currently wants its control + // planes to run, read from the same snapshot as everything else here. Only + // a process running exactly it may advance the pool's authorization + // projection - an older binary would otherwise publish its own older rules + // under a newer revision. + PublisherImage string + + // Frozen marks a pool whose desired configuration could not be resolved. + // Reconciliation then holds the last-good state: no creates, no drains, no + // deletes. It is NOT a desired count of zero, and it is NOT a startup + // failure — a bad ConfigMap must not take the control plane down. + Frozen bool + FrozenReason string +} + +func trinoPoolEnabled() bool { + enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(envTrinoPoolEnabled))) + return err == nil && enabled +} + +// trinoPoolOperatorEnabled reports whether this process may create, admit or +// delete instances. With the pool enabled but the operator off, the durable +// state is readable and nothing in Kubernetes or the Gateway is touched. +func trinoPoolOperatorEnabled() bool { + enabled, err := strconv.ParseBool(strings.TrimSpace(os.Getenv(envTrinoPoolOperatorEnabled))) + return err == nil && enabled +} + +// resolveTrinoPoolConfigs returns the shared pools declared in the registry. +// Cells without a pool block are ignored here and continue through the existing +// fixed-cell path untouched. +func resolveTrinoPoolConfigs() ([]trinoPoolConfig, error) { + return resolveTrinoPoolConfigsFrom(context.Background(), trinoPoolFileConfigReader{}) +} + +// resolveTrinoPoolConfigsFrom resolves every declared shared pool from one +// source. The source is the mounted files at startup and the ConfigMap the +// chart projects them from afterwards; the parsing and validation below are +// identical either way, so the two can never diverge in what they accept. +func resolveTrinoPoolConfigsFrom(ctx context.Context, reader trinoPoolConfigReader) ([]trinoPoolConfig, error) { + // ONE read. Everything below is parsed out of the same snapshot, so a + // resolution can never pair a registry entry with a blueprint the cluster + // no longer has - or the other way round. + snapshot, err := reader.Snapshot(ctx) + if err != nil { + return nil, err + } + data := snapshot.Registry() + if len(data) == 0 { + return nil, nil + } + cells, err := parseTrinoCellRegistry(data) + if err != nil { + return nil, err + } + + var configs []trinoPoolConfig + for _, cell := range cells { + mode := strings.TrimSpace(cell.Mode) + if mode == "" || mode == trinoPoolModeFixed { + continue + } + if mode != trinoPoolModeShared { + return nil, fmt.Errorf("Trino cell %s has an unsupported mode %q", cell.ID, mode) + } + // Asking for a pool while the feature is off must fail loudly. Falling + // back to fixed blue/green would give the operator a deployment shape + // they did not ask for and would not be told about. + if !trinoPoolEnabled() { + return nil, fmt.Errorf("Trino cell %s declares shared-pool mode but %s is not enabled", cell.ID, envTrinoPoolEnabled) + } + config, err := resolveTrinoPoolConfig(snapshot, cell) + if err != nil { + return nil, err + } + configs = append(configs, config) + } + return configs, nil +} + +// resolveTrinoPoolConfigByID re-resolves ONE pool's desired configuration from +// the authoritative source. +// +// This is what the operator calls before every desired-state publication. The +// source is the API object, not a mounted copy of it: a generation ordering +// cannot supply freshness on its own (a settings-only change need not move +// whatever produces that value, and two different configurations can carry the +// same one), and neither can re-reading a projected file, which lags per pod +// and, under a subPath mount, never updates at all. +// +// A pool that has DISAPPEARED from the registry is an error, not an empty +// configuration. Treating a vanished entry as "desired zero" would delete a +// running fleet because of a registry edit nobody meant as a teardown. +func resolveTrinoPoolConfigByID(ctx context.Context, reader trinoPoolConfigReader, publicID string) (trinoPoolConfig, error) { + configs, err := resolveTrinoPoolConfigsFrom(ctx, reader) + if err != nil { + return trinoPoolConfig{}, err + } + for _, config := range configs { + if config.PublicID == publicID { + return config, nil + } + } + return trinoPoolConfig{}, fmt.Errorf("Trino pool %s is no longer declared in %s", publicID, reader.Describe()) +} + +func resolveTrinoPoolConfig(snapshot trinoPoolConfigSnapshot, cell trinoRegisteredCell) (trinoPoolConfig, error) { + pool := cell.Pool + if pool == nil { + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s is in shared-pool mode but declares no pool block", cell.ID) + } + if len(cell.Backends) != 0 { + // A pooled cell's members are created by the operator and recorded in + // the config store. A static backend list would be a second, silently + // competing source of truth for the same routing group. + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s is in shared-pool mode and must not declare static backends", cell.ID) + } + if pool.DesiredInstances < 1 { + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s must declare at least one desired instance", cell.ID) + } + if pool.MinServing < 1 || pool.MinServing > pool.DesiredInstances { + // A floor above the desired count can never be satisfied, so every + // planned drain would be refused forever. + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s must keep its serving floor between one and the desired instance count", cell.ID) + } + if pool.MaxSurge < 0 || pool.MaxRepair < 0 { + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s declares a negative budget", cell.ID) + } + if pool.CoordinatorServicePort < 1 || pool.CoordinatorServicePort > 65535 { + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s declares an invalid coordinator service port", cell.ID) + } + if strings.TrimSpace(pool.NodeEnvironment) == "" { + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s declares no node environment", cell.ID) + } + if strings.TrimSpace(pool.BlueprintFile) == "" { + return trinoPoolConfig{}, fmt.Errorf("Trino cell %s declares no blueprint file", cell.ID) + } + + config := trinoPoolConfig{ + PoolID: registeredTrinoCellPrefix + cell.ID, + PublicID: cell.ID, + RoutingGroup: cell.RoutingGroup, + Namespace: cell.Namespace, + Pool: *pool, + // Read from the SAME snapshot as the registry and the blueprint, so the + // desired publisher and the desired configuration cannot be paired + // across an update. + PublisherImage: snapshot.PublisherImage(), + Spec: configstore.TrinoPoolSpec{ + PoolID: registeredTrinoCellPrefix + cell.ID, + PublicID: cell.ID, + APIMode: configstore.TrinoPoolAPIModeShared, + DesiredInstances: pool.DesiredInstances, + MinServing: pool.MinServing, + MaxSurge: pool.MaxSurge, + MaxRepair: pool.MaxRepair, + }, + } + + blueprint, err := loadTrinoPoolBlueprint(snapshot, pool.BlueprintFile, cell.Namespace) + if err != nil { + // Freeze rather than fail: the last-good desired state is preserved, + // the operator is told why, and nothing is created or deleted while the + // configuration is unreadable. + config.Frozen, config.FrozenReason = true, err.Error() + return config, nil + } + config.Blueprint = blueprint + config.Spec.DesiredReleaseID = blueprint.ReleaseID + config.Spec.DesiredBlueprintDigest = blueprint.Digest() + config.Spec.Generation = blueprint.Generation + return config, nil +} + +func loadTrinoPoolBlueprint(snapshot trinoPoolConfigSnapshot, declaredPath, namespace string) (*trinopool.Blueprint, error) { + data, err := snapshot.Blueprint(declaredPath) + if err != nil { + return nil, err + } + if len(data) > maxBlueprintFileBytes { + return nil, errors.New("blueprint exceeds the size limit") + } + blueprint, err := trinopool.ParseBlueprint(data) + if err != nil { + return nil, fmt.Errorf("blueprint is invalid: %w", err) + } + if blueprint.Namespace != namespace { + // Instances would land outside the namespace that holds the pool's + // shared Secrets, service accounts and network policy. + return nil, fmt.Errorf("blueprint targets namespace %q, the cell is in %q", blueprint.Namespace, namespace) + } + return blueprint, nil +} diff --git a/controlplane/trino_pool_config_source.go b/controlplane/trino_pool_config_source.go new file mode 100644 index 000000000..cf8d3a673 --- /dev/null +++ b/controlplane/trino_pool_config_source.go @@ -0,0 +1,240 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "fmt" + "os" + "path" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// Where the pool's desired configuration is READ from. +// +// A mounted file is not an authoritative answer to "what does the cluster want +// right now". The kubelet refreshes a projected ConfigMap volume on its own +// schedule, independently per pod, and a `subPath` mount is never refreshed at +// all - so two replicas can hold different contents indefinitely, and an idle +// pod can hold yesterday's. Re-reading such a file more often does not make it +// current; it only makes it re-read. +// +// The operator therefore publishes desired state from the API object itself. +// Files remain how the process BOOTS (they are what tells it a pool exists at +// all), and the API read is what every desired-state publication is derived +// from afterwards. The two inputs are the same documents: the registry the cell +// is declared in, and the blueprint it names. +const ( + envTrinoPoolConfigMap = "DUCKGRES_TRINO_POOL_CONFIG_CONFIGMAP" + envTrinoPoolConfigNS = "DUCKGRES_TRINO_POOL_CONFIG_NAMESPACE" + envTrinoPoolRegistryKey = "DUCKGRES_TRINO_POOL_CONFIG_REGISTRY_KEY" + defaultTrinoPoolRegistry = "cells.json" + + // One API read per pool per tick, from the leader only. The budget is short + // because a slow answer must not hold the reconcile loop: the pool keeps + // its last-good desired state and the next tick tries again. + trinoPoolConfigReadBudget = 10 * time.Second +) + +// trinoPoolConfigReader supplies one CONSISTENT set of desired-configuration +// documents. +// +// Snapshot is deliberately the only entry point. Reading the registry and the +// blueprint as two separate API calls can straddle an update and produce a +// configuration that never existed - a new registry entry paired with the +// previous release, say - and that mixture would be published as desired state. +// One object read once cannot do that. +type trinoPoolConfigReader interface { + // Snapshot reads every desired-configuration document at one instant. + Snapshot(ctx context.Context) (trinoPoolConfigSnapshot, error) + // Describe names the source in operator-facing errors. + Describe() string +} + +// trinoPoolConfigSnapshot is one point-in-time set of documents. +type trinoPoolConfigSnapshot interface { + // Registry returns the Trino cell registry document, or nil when this + // deployment declares no registry at all. + Registry() []byte + // Blueprint returns the blueprint the registry entry names. The argument is + // the declared blueprint path; a ConfigMap-backed snapshot uses its last + // element as the data key, which is exactly the mapping a ConfigMap volume + // mount performs, so one declaration addresses both sources. + Blueprint(declaredPath string) ([]byte, error) + // PublisherImage is the image the deployment currently WANTS to be running, + // rendered by the chart from the same helper as the Deployment's own image. + // It comes from this same snapshot so the desired publisher and the desired + // pool configuration are read together. Empty when the deployment does not + // declare one, which fails the projection fence closed. + PublisherImage() string +} + +// trinoPoolFileConfigReader reads the mounted documents. It is the BOOT source: +// it is what tells the process a pool exists. It is deliberately not used for +// desired-state publication. +type trinoPoolFileConfigReader struct{} + +// Snapshot reads the registry now and each blueprint when it is asked for. +// Files are the BOOT source only, where there is no desired-state publication +// to make inconsistent: the process is deciding whether a pool exists at all. +func (r trinoPoolFileConfigReader) Snapshot(context.Context) (trinoPoolConfigSnapshot, error) { + location := strings.TrimSpace(os.Getenv(envTrinoCellsFile)) + if location == "" { + return trinoPoolFileSnapshot{}, nil + } + data, err := os.ReadFile(location) + if err != nil { + return nil, fmt.Errorf("read Trino registry: %w", err) + } + return trinoPoolFileSnapshot{registry: data}, nil +} + +type trinoPoolFileSnapshot struct{ registry []byte } + +func (s trinoPoolFileSnapshot) Registry() []byte { return s.registry } + +// PublisherImage is empty for the mounted files: they are the BOOT source, and +// nothing is published from them. An empty value fails the projection fence +// closed, which is the correct answer for a source that cannot establish who +// the desired publisher is. +func (trinoPoolFileSnapshot) PublisherImage() string { return "" } + +func (trinoPoolFileSnapshot) Blueprint(declaredPath string) ([]byte, error) { + info, err := os.Stat(declaredPath) + if err != nil { + return nil, fmt.Errorf("blueprint is unreadable: %w", err) + } + if info.Size() > maxBlueprintFileBytes { + return nil, fmt.Errorf("blueprint exceeds the size limit") + } + data, err := os.ReadFile(declaredPath) + if err != nil { + return nil, fmt.Errorf("blueprint is unreadable: %w", err) + } + return data, nil +} + +func (trinoPoolFileConfigReader) Describe() string { return "the mounted configuration files" } + +// trinoPoolAPIConfigReader reads the SAME documents from the ConfigMap the +// chart projects them from, through the Kubernetes API. +// +// This is the authoritative source: the API object is one value shared by every +// replica, so two control planes cannot disagree about what is configured, and +// a pod that has been idle since boot reads what the cluster says now rather +// than what its kubelet last projected. +type trinoPoolAPIConfigReader struct { + client kubernetes.Interface + namespace string + name string + registryKey string +} + +func (r trinoPoolAPIConfigReader) Describe() string { + return fmt.Sprintf("ConfigMap %s/%s", r.namespace, r.name) +} + +// Snapshot reads the whole object ONCE. +// +// The registry and the blueprint are keys of the same ConfigMap and are taken +// from one read, so a resolution cannot pair a new registry entry with the +// previous release: a two-call reader can straddle an update and publish a +// configuration that never existed. No ordering is assumed between reads +// either - there is only one. +func (r trinoPoolAPIConfigReader) Snapshot(ctx context.Context) (trinoPoolConfigSnapshot, error) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolConfigReadBudget) + defer cancel() + + configMap, err := r.client.CoreV1().ConfigMaps(r.namespace).Get(ctx, r.name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("read %s: %w", r.Describe(), err) + } + snapshot := trinoPoolAPISnapshot{source: r.Describe(), data: map[string][]byte{}} + for key, value := range configMap.Data { + snapshot.data[key] = []byte(value) + } + for key, value := range configMap.BinaryData { + snapshot.data[key] = value + } + registry, err := snapshot.key(r.registryKey) + if err != nil { + return nil, err + } + snapshot.registry = registry + return snapshot, nil +} + +// trinoPoolAPISnapshot is one ConfigMap read, held as the complete set of +// documents that read contained. +type trinoPoolAPISnapshot struct { + source string + data map[string][]byte + registry []byte +} + +func (s trinoPoolAPISnapshot) Registry() []byte { return s.registry } + +// PublisherImage comes from the SAME read as the registry and the blueprint, so +// the desired publisher cannot be paired with a different snapshot of the +// desired configuration. +func (s trinoPoolAPISnapshot) PublisherImage() string { + return strings.TrimSpace(string(s.data[trinoPoolPublisherImageKey])) +} + +func (s trinoPoolAPISnapshot) Blueprint(declaredPath string) ([]byte, error) { + // A ConfigMap volume mounts each key as a file of that name, so the key is + // the declared path's last element. One registry declaration therefore + // addresses the file the process booted from AND the API object it is + // published from, with no second naming scheme to keep in sync. + return s.key(path.Base(strings.TrimSpace(declaredPath))) +} + +func (s trinoPoolAPISnapshot) key(key string) ([]byte, error) { + if key == "" { + return nil, fmt.Errorf("no key named in %s", s.source) + } + value, present := s.data[key] + if !present { + // An absent key is an error, never an empty document: an empty registry + // would read as "this pool no longer exists" and an empty blueprint as + // "no release", and neither is something a missing key may assert. + return nil, fmt.Errorf("%s has no key %q", s.source, key) + } + if len(value) > maxBlueprintFileBytes { + return nil, fmt.Errorf("%s key %q exceeds the size limit", s.source, key) + } + return value, nil +} + +// newTrinoPoolAPIConfigReader builds the authoritative reader for one pool. +// +// It is REQUIRED for a shared-pool cell. Falling back to the mounted files +// would mean publishing desired state from a per-pod snapshot that can lag +// indefinitely, which is the staleness this exists to remove - and a silent +// fallback is worse than a refusal, because nothing would say which source a +// running pool is being driven from. +func newTrinoPoolAPIConfigReader(client kubernetes.Interface, defaultNamespace string) (trinoPoolConfigReader, error) { + name := strings.TrimSpace(os.Getenv(envTrinoPoolConfigMap)) + if name == "" { + return nil, fmt.Errorf("a shared Trino pool requires %s: desired state is published from the ConfigMap, not from a mounted copy of it", envTrinoPoolConfigMap) + } + if client == nil { + return nil, fmt.Errorf("a shared Trino pool requires a Kubernetes client to read %s", name) + } + namespace := strings.TrimSpace(os.Getenv(envTrinoPoolConfigNS)) + if namespace == "" { + namespace = defaultNamespace + } + if namespace == "" { + return nil, fmt.Errorf("a shared Trino pool requires a namespace for %s", name) + } + registryKey := strings.TrimSpace(os.Getenv(envTrinoPoolRegistryKey)) + if registryKey == "" { + registryKey = defaultTrinoPoolRegistry + } + return trinoPoolAPIConfigReader{client: client, namespace: namespace, name: name, registryKey: registryKey}, nil +} diff --git a/controlplane/trino_pool_config_source_test.go b/controlplane/trino_pool_config_source_test.go new file mode 100644 index 000000000..2a2e8c967 --- /dev/null +++ b/controlplane/trino_pool_config_source_test.go @@ -0,0 +1,239 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "os" + "path/filepath" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// poolRegistryJSON is the registry document, parameterised by the values a +// settings-only edit would change. The image is deliberately NOT one of them: +// the case that matters is a configuration change that moves nothing an image +// or release ordering could order. +func poolRegistryJSON(blueprintPath string, desired, minServing int) string { + return `{"cells":[{ + "id":"cell-001", + "namespace":"trino-pool-example", + "client_url":"https://{database_name}.example.invalid", + "routing_group":"cell-001", + "mode":"shared-pool", + "pool":{ + "desired_instances":` + itoa(desired) + `, + "min_serving":` + itoa(minServing) + `, + "max_surge":1, + "max_repair":1, + "blueprint_file":"` + blueprintPath + `", + "coordinator_service_port":8443, + "node_environment":"mw_dev_pool_001" + } + }]}` +} + +func itoa(value int) string { + if value < 10 { + return string(rune('0' + value)) + } + return string(rune('0'+value/10)) + string(rune('0'+value%10)) +} + +// mountedRegistry writes ONE replica's projected copy of the configuration. +// Each call gets its own directory, because the point of these tests is that +// two replicas hold INDEPENDENT copies: a kubelet refreshes each pod's volume +// on its own schedule, and a subPath mount is never refreshed at all. +func mountedRegistry(t *testing.T, desired, minServing int) string { + t.Helper() + directory := t.TempDir() + blueprint := filepath.Join(directory, "blueprint.json") + if err := os.WriteFile(blueprint, testBlueprintJSON(t), 0o600); err != nil { + t.Fatalf("write blueprint: %v", err) + } + registry := filepath.Join(directory, "cells.json") + if err := os.WriteFile(registry, []byte(poolRegistryJSON(blueprint, desired, minServing)), 0o600); err != nil { + t.Fatalf("write registry: %v", err) + } + return registry +} + +// poolConfigMap is the authoritative object: ONE value the whole fleet reads, +// with the same keys a ConfigMap volume would project as files. +func poolConfigMap(t *testing.T, desired, minServing int) kubernetes.Interface { + t.Helper() + return fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "duckgres-trino-pool", Namespace: "trino-pool-example"}, + Data: map[string]string{ + "cells.json": poolRegistryJSON("/etc/duckgres/trino/blueprint.json", desired, minServing), + "blueprint.json": string(testBlueprintJSON(t)), + }, + }) +} + +func setPoolConfigMap(t *testing.T, client kubernetes.Interface, desired, minServing int) { + t.Helper() + configMap, err := client.CoreV1().ConfigMaps("trino-pool-example").Get(context.Background(), "duckgres-trino-pool", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get config map: %v", err) + } + configMap.Data["cells.json"] = poolRegistryJSON("/etc/duckgres/trino/blueprint.json", desired, minServing) + if _, err := client.CoreV1().ConfigMaps("trino-pool-example").Update(context.Background(), configMap, metav1.UpdateOptions{}); err != nil { + t.Fatalf("update config map: %v", err) + } +} + +func poolAPIReader(t *testing.T, client kubernetes.Interface) trinoPoolConfigReader { + t.Helper() + t.Setenv(envTrinoPoolConfigMap, "duckgres-trino-pool") + reader, err := newTrinoPoolAPIConfigReader(client, "trino-pool-example") + if err != nil { + t.Fatalf("build reader: %v", err) + } + return reader +} + +// Two replicas with independent projected copies read the SAME desired +// configuration, because both derive it from the API object rather than from +// their own mount. +// +// This is the case a per-tick file re-read cannot answer: replica A's volume +// still holds the previous configuration, and re-reading it more often does not +// make it current. Only the API object is one value for the whole fleet. +func TestDesiredStateComesFromTheAPIObjectNotTheMount(t *testing.T) { + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + client := poolConfigMap(t, 5, 4) + + // Replica A's mount is stale; replica B's is current but irrelevant. + staleMount := mountedRegistry(t, 3, 3) + freshMount := mountedRegistry(t, 5, 4) + + for _, mount := range []string{staleMount, freshMount} { + t.Setenv(envTrinoCellsFile, mount) + // What this replica would have published from its own copy. + mounted, err := resolveTrinoPoolConfigByID(context.Background(), trinoPoolFileConfigReader{}, "cell-001") + if err != nil { + t.Fatalf("resolve from the mount: %v", err) + } + resolved, err := resolveTrinoPoolConfigByID(context.Background(), poolAPIReader(t, client), "cell-001") + if err != nil { + t.Fatalf("resolve from the API: %v", err) + } + if resolved.Spec.DesiredInstances != 5 || resolved.Spec.MinServing != 4 { + t.Fatalf("resolved %d/%d from %s, want the API object's 5/4", + resolved.Spec.DesiredInstances, resolved.Spec.MinServing, mount) + } + if mount == staleMount && mounted.Spec.DesiredInstances == resolved.Spec.DesiredInstances { + t.Fatal("the stale mount and the API object agreed; this test is not exercising the divergence") + } + } +} + +// A configuration change is picked up without anything re-reading a file and +// without the process restarting: the next resolution reads the object again. +func TestDesiredStateFollowsTheAPIObjectAsItChanges(t *testing.T) { + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + t.Setenv(envTrinoCellsFile, mountedRegistry(t, 3, 3)) + client := poolConfigMap(t, 3, 3) + reader := poolAPIReader(t, client) + + first, err := resolveTrinoPoolConfigByID(context.Background(), reader, "cell-001") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if first.Spec.DesiredInstances != 3 { + t.Fatalf("desired instances = %d, want 3", first.Spec.DesiredInstances) + } + + setPoolConfigMap(t, client, 5, 4) + + second, err := resolveTrinoPoolConfigByID(context.Background(), reader, "cell-001") + if err != nil { + t.Fatalf("re-resolve: %v", err) + } + if second.Spec.DesiredInstances != 5 || second.Spec.MinServing != 4 { + t.Fatalf("desired = %d/%d, want the updated 5/4", second.Spec.DesiredInstances, second.Spec.MinServing) + } +} + +// An unreadable object is an error the caller turns into a freeze. It must +// never resolve to an empty configuration, because "no pools" and "desired +// zero" would delete a running fleet over an API blip. +func TestUnreadableDesiredStateSourceIsAnError(t *testing.T) { + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + client := fake.NewClientset() + if _, err := resolveTrinoPoolConfigByID(context.Background(), poolAPIReader(t, client), "cell-001"); err == nil { + t.Fatal("a missing ConfigMap resolved successfully") + } + + missingKey := fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "duckgres-trino-pool", Namespace: "trino-pool-example"}, + Data: map[string]string{"cells.json": poolRegistryJSON("/etc/duckgres/trino/blueprint.json", 3, 3)}, + }) + // A registry that resolves but whose blueprint does not is the FROZEN case, + // not the error case: the pool is known, its release is not, so it holds + // its last-good state instead of being reshaped by a half-readable source. + config, err := resolveTrinoPoolConfigByID(context.Background(), poolAPIReader(t, missingKey), "cell-001") + if err != nil { + t.Fatalf("resolve with an unreadable blueprint: %v", err) + } + if !config.Frozen || config.Blueprint != nil { + t.Fatalf("config = %+v, want a frozen pool with no blueprint", config) + } +} + +// The blueprint key is the declared path's last element, which is exactly the +// mapping a ConfigMap volume performs. One declaration therefore addresses the +// file this process booted from and the object it publishes from, with no +// second naming scheme that could drift. +func TestBlueprintKeyIsTheDeclaredFileName(t *testing.T) { + reader := poolAPIReader(t, poolConfigMap(t, 3, 3)) + snapshot, err := reader.Snapshot(context.Background()) + if err != nil { + t.Fatalf("snapshot: %v", err) + } + data, err := snapshot.Blueprint("/etc/duckgres/trino/blueprint.json") + if err != nil { + t.Fatalf("read blueprint: %v", err) + } + if len(data) == 0 { + t.Fatal("the blueprint key resolved to nothing") + } +} + +// The registry and the blueprint come from ONE read of ONE object. +// +// Reading them as two calls can straddle an update and pair a new registry +// entry with the release the cluster has already replaced - a configuration +// that never existed anywhere, published as desired state. The reader is +// asserted to make a single Get per resolution. +func TestDesiredStateIsOneAtomicRead(t *testing.T) { + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + client := poolConfigMap(t, 3, 3) + gets := 0 + fakeClient, ok := client.(*fake.Clientset) + if !ok { + t.Fatalf("unexpected client type %T", client) + } + fakeClient.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + gets++ + return false, nil, nil + }) + + if _, err := resolveTrinoPoolConfigByID(context.Background(), poolAPIReader(t, client), "cell-001"); err != nil { + t.Fatalf("resolve: %v", err) + } + if gets != 1 { + t.Fatalf("the resolution made %d reads of the desired-state object, want exactly 1", gets) + } +} diff --git a/controlplane/trino_pool_config_test.go b/controlplane/trino_pool_config_test.go new file mode 100644 index 000000000..357752b78 --- /dev/null +++ b/controlplane/trino_pool_config_test.go @@ -0,0 +1,224 @@ +//go:build kubernetes + +package controlplane + +import ( + "os" + "path/filepath" + "testing" +) + +func writeRegistry(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "cells.json") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("write registry: %v", err) + } + return path +} + +func sharedPoolRegistry(t *testing.T, blueprintPath string) string { + t.Helper() + return writeRegistry(t, `{"cells":[{ + "id":"cell-001", + "namespace":"trino-cell-001", + "client_url":"https://{database_name}.example.invalid", + "routing_group":"cell-001", + "mode":"shared-pool", + "pool":{ + "desired_instances":3, + "min_serving":3, + "max_surge":1, + "max_repair":1, + "blueprint_file":"`+blueprintPath+`", + "coordinator_service_port":8443, + "node_environment":"mw_dev_pool_001" + } + }]}`) +} + +func blueprintFile(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "blueprint.json") + if err := os.WriteFile(path, testBlueprintJSON(t), 0o600); err != nil { + t.Fatalf("write blueprint: %v", err) + } + return path +} + +func withPoolEnv(t *testing.T, values map[string]string) { + t.Helper() + for name, value := range values { + t.Setenv(name, value) + } +} + +// A registry that declares a shared pool while the feature flag is off must +// fail startup, not silently fall back to legacy behavior: the operator asked +// for a pool and would otherwise get fixed blue/green without being told. +func TestSharedPoolCellRequiresTheFeatureFlag(t *testing.T) { + withPoolEnv(t, map[string]string{ + envTrinoCellsFile: sharedPoolRegistry(t, blueprintFile(t)), + envTrinoRegistryOnly: "true", + envTrinoPoolEnabled: "", + }) + if _, err := resolveTrinoPoolConfigs(); err == nil { + t.Fatal("a shared-pool cell was accepted with the feature disabled") + } +} + +func TestSharedPoolConfigResolves(t *testing.T) { + withPoolEnv(t, map[string]string{ + envTrinoCellsFile: sharedPoolRegistry(t, blueprintFile(t)), + envTrinoRegistryOnly: "true", + envTrinoPoolEnabled: "true", + }) + configs, err := resolveTrinoPoolConfigs() + if err != nil { + t.Fatalf("resolve: %v", err) + } + if len(configs) != 1 { + t.Fatalf("resolved %d pools", len(configs)) + } + config := configs[0] + // The pool keeps the cell's existing LOGICAL ASSIGNMENT identity: the pool + // id, the routing group and the namespace are all unchanged, because compute + // replacement must not rewrite warehouse assignment. The catalog store's + // partition is a separate, separately configured identity and is resolved + // nowhere near here. + if config.PoolID != registeredTrinoCellPrefix+"cell-001" || config.PublicID != "cell-001" { + t.Fatalf("pool identity = %q / %q", config.PoolID, config.PublicID) + } + if config.RoutingGroup != "cell-001" || config.Namespace != "trino-cell-001" { + t.Fatalf("routing group / namespace = %q / %q", config.RoutingGroup, config.Namespace) + } + if config.Spec.DesiredInstances != 3 || config.Spec.MinServing != 3 || config.Spec.MaxSurge != 1 || config.Spec.MaxRepair != 1 { + t.Fatalf("spec = %+v", config.Spec) + } +} + +// A cell with no mode is the existing fixed blue/green cell, byte for byte. +func TestFixedCellsAreNotPools(t *testing.T) { + withPoolEnv(t, map[string]string{ + envTrinoRegistryOnly: "true", + envTrinoPoolEnabled: "true", + envTrinoCellsFile: writeRegistry(t, `{"cells":[{ + "id":"cell-002","namespace":"trino-cell-002", + "client_url":"https://{database_name}.example.invalid","routing_group":"cell-002", + "backends":[ + {"id":"blue","coordinator_url":"https://blue.invalid","running":true,"routing_active":true,"internal_secret_name":"blue-secret"}, + {"id":"green","coordinator_url":"https://green.invalid","running":false,"routing_active":false,"internal_secret_name":"green-secret"} + ]}]}`), + }) + configs, err := resolveTrinoPoolConfigs() + if err != nil { + t.Fatalf("resolve: %v", err) + } + if len(configs) != 0 { + t.Fatalf("a fixed cell produced %d pools", len(configs)) + } +} + +func TestSharedPoolRegistryValidation(t *testing.T) { + blueprint := blueprintFile(t) + cases := map[string]string{ + "backends on a pooled cell": `{"cells":[{"id":"cell-001","namespace":"trino-cell-001", + "client_url":"https://{database_name}.example.invalid","routing_group":"cell-001","mode":"shared-pool", + "pool":{"desired_instances":3,"min_serving":3,"max_surge":1,"max_repair":1,"blueprint_file":"` + blueprint + `","coordinator_service_port":8443,"node_environment":"e"}, + "backends":[{"id":"blue","coordinator_url":"https://blue.invalid","running":true,"routing_active":true,"internal_secret_name":"s"}]}]}`, + "missing pool block": `{"cells":[{"id":"cell-001","namespace":"trino-cell-001", + "client_url":"https://{database_name}.example.invalid","routing_group":"cell-001","mode":"shared-pool"}]}`, + "unknown mode": `{"cells":[{"id":"cell-001","namespace":"trino-cell-001", + "client_url":"https://{database_name}.example.invalid","routing_group":"cell-001","mode":"elastic", + "pool":{"desired_instances":3,"min_serving":3,"max_surge":1,"max_repair":1,"blueprint_file":"` + blueprint + `","coordinator_service_port":8443,"node_environment":"e"}}]}`, + // A serving floor above the desired count can never be satisfied, so + // the pool would refuse every planned drain forever. + "floor above desired": `{"cells":[{"id":"cell-001","namespace":"trino-cell-001", + "client_url":"https://{database_name}.example.invalid","routing_group":"cell-001","mode":"shared-pool", + "pool":{"desired_instances":2,"min_serving":3,"max_surge":1,"max_repair":1,"blueprint_file":"` + blueprint + `","coordinator_service_port":8443,"node_environment":"e"}}]}`, + "zero desired instances": `{"cells":[{"id":"cell-001","namespace":"trino-cell-001", + "client_url":"https://{database_name}.example.invalid","routing_group":"cell-001","mode":"shared-pool", + "pool":{"desired_instances":0,"min_serving":0,"max_surge":1,"max_repair":1,"blueprint_file":"` + blueprint + `","coordinator_service_port":8443,"node_environment":"e"}}]}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + withPoolEnv(t, map[string]string{ + envTrinoCellsFile: writeRegistry(t, body), + envTrinoRegistryOnly: "true", + envTrinoPoolEnabled: "true", + }) + if _, err := resolveTrinoPoolConfigs(); err == nil { + t.Fatalf("accepted %s", name) + } + }) + } +} + +// An unreadable or invalid blueprint FREEZES the pool at its last-good state. +// It must never resolve to a desired count of zero, which would delete the +// fleet, and it must not abort startup, which would take the control plane +// down over a config-map problem. +func TestUnreadableBlueprintFreezesTheConfig(t *testing.T) { + missing := filepath.Join(t.TempDir(), "absent.json") + withPoolEnv(t, map[string]string{ + envTrinoCellsFile: sharedPoolRegistry(t, missing), + envTrinoRegistryOnly: "true", + envTrinoPoolEnabled: "true", + }) + configs, err := resolveTrinoPoolConfigs() + if err != nil { + t.Fatalf("resolve: %v", err) + } + if len(configs) != 1 { + t.Fatalf("resolved %d pools", len(configs)) + } + config := configs[0] + if !config.Frozen || config.FrozenReason == "" { + t.Fatal("an unreadable blueprint did not freeze the pool") + } + if config.Blueprint != nil { + t.Fatal("a frozen pool carries a blueprint") + } + if config.Spec.DesiredInstances == 0 { + t.Fatal("a frozen pool resolved to a desired count of zero") + } +} + +func TestInvalidBlueprintFreezesTheConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "blueprint.json") + if err := os.WriteFile(path, []byte(`{"blueprint_version":1}`), 0o600); err != nil { + t.Fatalf("write blueprint: %v", err) + } + withPoolEnv(t, map[string]string{ + envTrinoCellsFile: sharedPoolRegistry(t, path), + envTrinoRegistryOnly: "true", + envTrinoPoolEnabled: "true", + }) + configs, err := resolveTrinoPoolConfigs() + if err != nil { + t.Fatalf("resolve: %v", err) + } + if !configs[0].Frozen { + t.Fatal("an invalid blueprint did not freeze the pool") + } +} + +// The blueprint's namespace and the cell's namespace must agree, or duckgres +// would create instances somewhere the pool's shared trust boundary does not +// exist. +func TestBlueprintNamespaceMustMatchTheCell(t *testing.T) { + withPoolEnv(t, map[string]string{ + envTrinoCellsFile: writeRegistry(t, `{"cells":[{"id":"cell-009","namespace":"trino-cell-009", + "client_url":"https://{database_name}.example.invalid","routing_group":"cell-009","mode":"shared-pool", + "pool":{"desired_instances":3,"min_serving":3,"max_surge":1,"max_repair":1,"blueprint_file":"`+blueprintFile(t)+`","coordinator_service_port":8443,"node_environment":"e"}}]}`), + envTrinoRegistryOnly: "true", + envTrinoPoolEnabled: "true", + }) + configs, err := resolveTrinoPoolConfigs() + if err != nil { + t.Fatalf("resolve: %v", err) + } + if !configs[0].Frozen { + t.Fatal("a blueprint for another namespace was accepted") + } +} diff --git a/controlplane/trino_pool_durable.go b/controlplane/trino_pool_durable.go new file mode 100644 index 000000000..18e3c4e78 --- /dev/null +++ b/controlplane/trino_pool_durable.go @@ -0,0 +1,190 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "math/rand/v2" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" +) + +// Durable operations around external effects. +// +// Every Gateway mutation is preceded by a recorded intent and followed by a +// recorded outcome. That is what makes a LOST response recoverable: the next +// attempt - possibly by a different leader - reads the operation back under the +// same identity instead of guessing whether the effect happened. +// +// The Gateway has its own replay guard, so a repeated identical call is already +// safe there. What it cannot do is tell THIS controller what a previous leader +// attempted, which is why the record lives here too. + +// trinoPoolOperationStore is the durable-operation surface the operator uses. +type trinoPoolOperationStore interface { + BeginTrinoPoolOperation(context.Context, configstore.TrinoPoolLease, configstore.TrinoPoolOperationSpec) (configstore.TrinoPoolOperation, error) + RecordTrinoPoolOperationStep(ctx context.Context, lease configstore.TrinoPoolLease, operationID, stepID, payloadHash, outcome, result string) (configstore.TrinoPoolOperationStep, error) + FinishTrinoPoolOperation(ctx context.Context, lease configstore.TrinoPoolLease, operationID, phase, lastError string) error + UpdateTrinoPoolOperation(ctx context.Context, lease configstore.TrinoPoolLease, operationID string, updates map[string]any) error +} + +// Durable retry pacing. A failing external call is retried on a schedule the +// DATABASE holds, not the leader's memory: a restart or a leadership move would +// otherwise reset every backoff to zero and turn a persistent failure into a +// hot loop against the Gateway. +const ( + trinoPoolRetryBase = 500 * time.Millisecond + trinoPoolRetryMax = 30 * time.Second +) + +// errTrinoPoolBackoff means "not yet" - the operation has a recorded next +// attempt in the future. It is not a failure: nothing was attempted, and the +// reconcile loop treats it as a quiet no-op rather than an error to alert on. +var errTrinoPoolBackoff = errors.New("trino pool operation is waiting for its next attempt") + +// trinoPoolRetryDelay is full jitter over an exponential backoff: every attempt +// waits a random duration up to the exponential bound, so several controllers +// retrying the same class of failure do not synchronize into bursts. +func trinoPoolRetryDelay(attempts int64) time.Duration { + bound := trinoPoolRetryBase + for i := int64(0); i < attempts && bound < trinoPoolRetryMax; i++ { + bound *= 2 + } + if bound > trinoPoolRetryMax { + bound = trinoPoolRetryMax + } + return time.Duration(rand.Int64N(int64(bound)) + int64(trinoPoolRetryBase)) +} + +// Recorded step outcomes. +const ( + trinoPoolStepOK = configstore.TrinoPoolStepOutcomeOK + trinoPoolStepUnknown = configstore.TrinoPoolStepOutcomeUnknown + trinoPoolStepFailed = configstore.TrinoPoolStepOutcomeFailed +) + +// runDurableStep records the intent, performs the effect, and records what came +// back. +// +// The three outcomes are deliberately distinct: +// +// - OK: the Gateway answered. The recorded result is the answer. +// - FAILED: the Gateway REFUSED. That is a decision; retrying cannot change +// it, and recording it as unknown would invite exactly that retry. +// - UNKNOWN: no answer arrived. The effect may or may not have happened, so +// the step stays open and the next attempt resolves it by read-back under +// the same identity rather than by repeating blind. +func (o *trinoPoolOperator) runDurableStep( + ctx context.Context, + operation configstore.TrinoPoolOperationSpec, + stepID string, + payload any, + effect func(context.Context) (string, error), +) error { + if o.operations == nil { + // Durable recording is unavailable (tests, or a store that does not + // implement it). The effect still runs: refusing to act because the + // journal is missing would be a worse failure than acting unrecorded. + _, err := effect(ctx) + return err + } + + recordedOperation, err := o.operations.BeginTrinoPoolOperation(ctx, o.lease, operation) + if err != nil { + return o.dropAuthority(fmt.Errorf("record intent for %s: %w", operation.OperationID, err)) + } + if recordedOperation.NextAttemptAt != nil && time.Now().UTC().Before(*recordedOperation.NextAttemptAt) { + // A previous attempt failed and the wait it earned has not elapsed. + // Retrying now would hammer whatever refused it, and the schedule is + // durable precisely so a restart cannot skip it. + return fmt.Errorf("%w: %s until %s", errTrinoPoolBackoff, + operation.OperationID, recordedOperation.NextAttemptAt.Format(time.RFC3339)) + } + + hash := trinoPoolPayloadHash(payload) + recorded, err := o.operations.RecordTrinoPoolOperationStep(ctx, o.lease, + operation.OperationID, stepID, hash, trinoPoolStepUnknown, "{}") + if err != nil { + if errors.Is(err, configstore.ErrTrinoPoolIntentChanged) { + // The same step id was already recorded with different content. + // Performing this effect would apply an intent nobody recorded. + return fmt.Errorf("step %s of %s was recorded with different content: %w", + stepID, operation.OperationID, err) + } + return o.dropAuthority(fmt.Errorf("record step %s: %w", stepID, err)) + } + if recorded.Replayed && recorded.Outcome == trinoPoolStepOK { + // A previous attempt - possibly by another leader - already completed + // this step. Repeating the effect is unnecessary; the Gateway would + // replay it anyway, but not calling at all is cheaper and clearer. + slog.Debug("Trino pool step already completed.", + "pool", o.config.PublicID, "operation", operation.OperationID, "step", stepID) + return nil + } + + result, effectErr := effect(ctx) + outcome := trinoPoolStepOK + if effectErr != nil { + outcome = trinoPoolStepUnknown + if trinoPoolDecided(effectErr) { + outcome = trinoPoolStepFailed + } + } + if _, err := o.operations.RecordTrinoPoolOperationStep(ctx, o.lease, + operation.OperationID, stepID, hash, outcome, result); err != nil && + !errors.Is(err, configstore.ErrTrinoPoolIntentChanged) { + slog.Warn("Trino pool step outcome could not be recorded.", + "pool", o.config.PublicID, "operation", operation.OperationID, "step", stepID, "error", err) + } + o.recordAttempt(ctx, operation.OperationID, recordedOperation.Attempts, effectErr) + return effectErr +} + +// recordAttempt persists the retry schedule, or closes the operation when the +// effect succeeded. +// +// Both halves matter. Without the schedule, `attempts` and `next_attempt_at` +// stay untouched and every failing call is retried on every tick forever; with +// no terminal marker, the operations table only grows and nothing can +// distinguish work in flight from work that finished. +func (o *trinoPoolOperator) recordAttempt(ctx context.Context, operationID string, attempts int64, effectErr error) { + if effectErr == nil { + if err := o.operations.FinishTrinoPoolOperation(ctx, o.lease, operationID, "completed", ""); err != nil && + !errors.Is(err, configstore.ErrTrinoPoolConflict) { + slog.Warn("Trino pool operation could not be closed.", + "pool", o.config.PublicID, "operation", operationID, "error", err) + } + return + } + next := time.Now().UTC().Add(trinoPoolRetryDelay(attempts)) + if err := o.operations.UpdateTrinoPoolOperation(ctx, o.lease, operationID, map[string]any{ + "attempts": attempts + 1, + "next_attempt_at": next, + "last_error": effectErr.Error(), + }); err != nil && !errors.Is(err, configstore.ErrTrinoPoolConflict) { + slog.Warn("Trino pool retry schedule could not be recorded.", + "pool", o.config.PublicID, "operation", operationID, "error", err) + } +} + +// trinoPoolDecided reports whether the Gateway made a decision, as opposed to +// never answering. A decision is terminal for the step; an unanswered call is +// not, and the difference is what keeps a retry loop from hiding a refusal. +func trinoPoolDecided(err error) bool { + var gatewayError *trinogateway.Error + return errors.As(err, &gatewayError) +} + +// trinoPoolPayloadHash identifies a step's intent, so a replay with different +// content is recognizable as a conflict rather than applied silently. +func trinoPoolPayloadHash(payload any) string { + digest := sha256.Sum256([]byte(fmt.Sprintf("%#v", payload))) + return hex.EncodeToString(digest[:]) +} diff --git a/controlplane/trino_pool_effects.go b/controlplane/trino_pool_effects.go new file mode 100644 index 000000000..31935dd3b --- /dev/null +++ b/controlplane/trino_pool_effects.go @@ -0,0 +1,532 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/posthog/duckgres/controlplane/trinopool" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +// Kubernetes effects for one shared-pool instance. +// +// There is no atomic handover across the duckgres database, the Gateway +// database and Kubernetes, so every write here is conditional and every delete +// carries a precondition. The three rules: +// +// - Never adopt an object that is not ours. On AlreadyExists we compare +// ownership and identity; a foreign or mismatched object is an error, not a +// thing to overwrite. +// - Never lower the authority epoch. A superseded leader that has not noticed +// yet is refused at the object level, not just at the loop entry. +// - Never delete by name alone. A UID precondition is what stops a delete from +// removing a replacement object that happens to share the name. +const ( + trinoPoolRequestBudget = 10 * time.Second + trinoPoolListLimit = 500 +) + +var ( + errTrinoPoolForeignObject = errors.New("kubernetes object is not managed by this trino pool") + errTrinoPoolStaleEpoch = errors.New("kubernetes object carries a newer authority epoch") + errTrinoPoolSharedResource = errors.New("refusing to touch a pool-shared resource") +) + +// trinoPoolInventory is the recorded Kubernetes identity of one instance. It is +// persisted before any delete, so a delete always knows exactly what it may +// remove. +type trinoPoolInventory struct { + Namespace string + ConfigMapName string + ConfigMapUID string + WorkerConfigMapName string + WorkerConfigMapUID string + ServiceName string + ServiceUID string + CoordinatorDeploymentName string + CoordinatorDeploymentUID string + WorkerDeploymentName string + WorkerDeploymentUID string +} + +// trinoPoolObservation is what the cluster currently reports about an instance. +type trinoPoolObservation struct { + ReadyWorkers int + DesiredWorkers int + CoordinatorReady bool + CoordinatorPodUID string + PodsPresent int + // The images the pods are ACTUALLY running, read from the pod specs. The + // admission receipt claims an image check, and a claim that compares + // nothing is a false statement in the Gateway's durable evidence. + CoordinatorImage string + WorkerImage string + // CoordinatorPods is every coordinator pod of this instance, terminating + // ones included, with the container-level facts a loss claim needs. A + // coordinator that restarts in place keeps all of its objects, so absence + // can never become evidence for it; what CAN is Kubernetes' own record that + // the container hosting the admitted process ended. + CoordinatorPods []trinoPoolCoordinatorPod +} + +// trinoPoolCoordinatorPod is one coordinator pod as the cluster reports it. +type trinoPoolCoordinatorPod struct { + UID string + Terminating bool + // RunningContainerID, Restarts and LastTerminated come from the pod's own + // container status for the Trino container. The kubelet writes them AFTER it + // observed the exit, so they are a statement about a container that ended, + // never a timeout. The IDs are what tie a record to ONE container instance: + // a termination that names some other instance says nothing about the + // admitted one. + RunningContainerID string + Restarts int32 + LastTerminated *trinoPoolContainerTermination +} + +// trinoPoolContainerTermination is Kubernetes' record of a container instance +// that ended, carried into the loss claim so the evidence names what was seen. +type trinoPoolContainerTermination struct { + ContainerID string + ExitCode int32 + Reason string + FinishedAt string +} + +type trinoPoolEffects struct { + clientset kubernetes.Interface + namespace string + shared trinopool.BlueprintSharedResources + epoch int64 +} + +func newTrinoPoolEffects(clientset kubernetes.Interface, namespace string, shared trinopool.BlueprintSharedResources, epoch int64) *trinoPoolEffects { + return &trinoPoolEffects{clientset: clientset, namespace: namespace, shared: shared, epoch: epoch} +} + +// Apply creates or adopts this instance's objects and returns the recorded +// inventory. It is idempotent: a lost create response is resolved by reading the +// deterministic name back, never by creating a second instance. +func (e *trinoPoolEffects) Apply(ctx context.Context, objects trinopool.Objects) (trinoPoolInventory, error) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolRequestBudget) + defer cancel() + + inventory := trinoPoolInventory{Namespace: e.namespace} + + configMapUID, err := e.applyConfigMap(ctx, objects.ConfigMap) + if err != nil { + return inventory, err + } + inventory.ConfigMapName, inventory.ConfigMapUID = objects.ConfigMap.Name, configMapUID + + workerConfigUID, err := e.applyConfigMap(ctx, objects.WorkerConfigMap) + if err != nil { + return inventory, err + } + inventory.WorkerConfigMapName, inventory.WorkerConfigMapUID = objects.WorkerConfigMap.Name, workerConfigUID + + serviceUID, err := e.applyService(ctx, objects.Service) + if err != nil { + return inventory, err + } + inventory.ServiceName, inventory.ServiceUID = objects.Service.Name, serviceUID + + // The coordinator is created before the workers so the discovery endpoint + // exists by the time a worker tries to register. + coordinatorUID, err := e.applyDeployment(ctx, objects.CoordinatorDeployment) + if err != nil { + return inventory, err + } + inventory.CoordinatorDeploymentName, inventory.CoordinatorDeploymentUID = objects.CoordinatorDeployment.Name, coordinatorUID + + workerUID, err := e.applyDeployment(ctx, objects.WorkerDeployment) + if err != nil { + return inventory, err + } + inventory.WorkerDeploymentName, inventory.WorkerDeploymentUID = objects.WorkerDeployment.Name, workerUID + + return inventory, nil +} + +func (e *trinoPoolEffects) applyConfigMap(ctx context.Context, desired *corev1.ConfigMap) (string, error) { + if err := e.guard(desired.Name); err != nil { + return "", err + } + created, err := e.clientset.CoreV1().ConfigMaps(e.namespace).Create(ctx, desired, metav1.CreateOptions{}) + if err == nil { + return string(created.UID), nil + } + if !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("create config map %s: %w", desired.Name, err) + } + existing, err := e.clientset.CoreV1().ConfigMaps(e.namespace).Get(ctx, desired.Name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("read back config map %s: %w", desired.Name, err) + } + if err := e.checkOwnership(existing.ObjectMeta, desired.ObjectMeta); err != nil { + return "", err + } + return string(existing.UID), nil +} + +func (e *trinoPoolEffects) applyService(ctx context.Context, desired *corev1.Service) (string, error) { + if err := e.guard(desired.Name); err != nil { + return "", err + } + created, err := e.clientset.CoreV1().Services(e.namespace).Create(ctx, desired, metav1.CreateOptions{}) + if err == nil { + return string(created.UID), nil + } + if !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("create service %s: %w", desired.Name, err) + } + existing, err := e.clientset.CoreV1().Services(e.namespace).Get(ctx, desired.Name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("read back service %s: %w", desired.Name, err) + } + if err := e.checkOwnership(existing.ObjectMeta, desired.ObjectMeta); err != nil { + return "", err + } + return string(existing.UID), nil +} + +func (e *trinoPoolEffects) applyDeployment(ctx context.Context, desired *appsv1.Deployment) (string, error) { + if err := e.guard(desired.Name); err != nil { + return "", err + } + created, err := e.clientset.AppsV1().Deployments(e.namespace).Create(ctx, desired, metav1.CreateOptions{}) + if err == nil { + return string(created.UID), nil + } + if !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("create deployment %s: %w", desired.Name, err) + } + existing, err := e.clientset.AppsV1().Deployments(e.namespace).Get(ctx, desired.Name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("read back deployment %s: %w", desired.Name, err) + } + if err := e.checkOwnership(existing.ObjectMeta, desired.ObjectMeta); err != nil { + return "", err + } + // An existing object with OUR identity and the same spec digest is this + // operation's own earlier attempt. It is adopted as-is and never patched: + // a serving instance's execution configuration is immutable by design. + return string(existing.UID), nil +} + +// checkOwnership decides whether an existing object is this instance's own. +func (e *trinoPoolEffects) checkOwnership(existing, desired metav1.ObjectMeta) error { + if existing.Labels[trinopool.LabelManagedBy] != trinopool.ManagedByValue || + existing.Labels[trinopool.LabelInstance] != desired.Labels[trinopool.LabelInstance] { + return fmt.Errorf("%w: %s", errTrinoPoolForeignObject, existing.Name) + } + recorded, err := strconv.ParseInt(existing.Annotations[trinopool.AnnotationAuthorityEpoch], 10, 64) + if err != nil { + return fmt.Errorf("%w: %s has no readable authority epoch", errTrinoPoolForeignObject, existing.Name) + } + if recorded > e.epoch { + return fmt.Errorf("%w: %s is owned at epoch %d, this leader holds %d", errTrinoPoolStaleEpoch, existing.Name, recorded, e.epoch) + } + if existing.Annotations[trinopool.AnnotationSpecDigest] != desired.Annotations[trinopool.AnnotationSpecDigest] { + return fmt.Errorf("%w: %s carries a different spec digest", errTrinoPoolForeignObject, existing.Name) + } + return nil +} + +// guard refuses to touch anything the pool shares. Instance cleanup must never +// be able to remove the pool's trust boundary, so the check is here rather than +// only in the caller. +func (e *trinoPoolEffects) guard(name string) error { + if e.shared.Protects(name) { + return fmt.Errorf("%w: %s", errTrinoPoolSharedResource, name) + } + return nil +} + +// Delete removes the instance's objects. Controllers go first: deleting pods +// while their Deployment still exists just produces new pods. +// +// The caller must hold an irreversible retirement receipt for this exact +// incarnation before calling this. Nothing here checks that, because nothing +// here can - it is enforced by the instance phase, which only permits deletion +// from RETIRING onwards. +func (e *trinoPoolEffects) Delete(ctx context.Context, inventory trinoPoolInventory) error { + ctx, cancel := context.WithTimeout(ctx, trinoPoolRequestBudget) + defer cancel() + + for _, name := range []string{ + inventory.CoordinatorDeploymentName, inventory.WorkerDeploymentName, + inventory.ServiceName, inventory.ConfigMapName, inventory.WorkerConfigMapName, + } { + if err := e.guard(name); err != nil { + return err + } + } + + deploymentUID := func(ctx context.Context, name string) (string, error) { + object, err := e.clientset.AppsV1().Deployments(inventory.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return "", err + } + return string(object.UID), nil + } + serviceUID := func(ctx context.Context, name string) (string, error) { + object, err := e.clientset.CoreV1().Services(inventory.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return "", err + } + return string(object.UID), nil + } + configMapUID := func(ctx context.Context, name string) (string, error) { + object, err := e.clientset.CoreV1().ConfigMaps(inventory.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return "", err + } + return string(object.UID), nil + } + deleteDeployment := func(ctx context.Context, name string, options metav1.DeleteOptions) error { + return e.clientset.AppsV1().Deployments(inventory.Namespace).Delete(ctx, name, options) + } + deleteService := func(ctx context.Context, name string, options metav1.DeleteOptions) error { + return e.clientset.CoreV1().Services(inventory.Namespace).Delete(ctx, name, options) + } + deleteConfigMap := func(ctx context.Context, name string, options metav1.DeleteOptions) error { + return e.clientset.CoreV1().ConfigMaps(inventory.Namespace).Delete(ctx, name, options) + } + + // Workload controllers first: deleting pods while their Deployment still + // exists just makes the Deployment create new ones. + deletions := []struct { + kind string + name string + uid string + observe func(context.Context, string) (string, error) + call func(context.Context, string, metav1.DeleteOptions) error + }{ + {"deployment", inventory.CoordinatorDeploymentName, inventory.CoordinatorDeploymentUID, deploymentUID, deleteDeployment}, + {"deployment", inventory.WorkerDeploymentName, inventory.WorkerDeploymentUID, deploymentUID, deleteDeployment}, + {"service", inventory.ServiceName, inventory.ServiceUID, serviceUID, deleteService}, + {"config map", inventory.ConfigMapName, inventory.ConfigMapUID, configMapUID, deleteConfigMap}, + {"config map", inventory.WorkerConfigMapName, inventory.WorkerConfigMapUID, configMapUID, deleteConfigMap}, + } + for _, deletion := range deletions { + if deletion.name == "" { + continue + } + // Verify the UID ourselves before asking the API server to, so the + // check holds even against an implementation that ignores + // preconditions. A name whose UID has moved on belongs to a different + // object, and deleting it would destroy somebody else's workload. + current, err := deletion.observe(ctx, deletion.name) + switch { + case apierrors.IsNotFound(err): + continue + case err != nil: + return fmt.Errorf("read back %s %s: %w", deletion.kind, deletion.name, err) + case deletion.uid != "" && current != deletion.uid: + return fmt.Errorf("%w: %s %s is now %s, recorded %s", + errTrinoPoolForeignObject, deletion.kind, deletion.name, current, deletion.uid) + } + + options := metav1.DeleteOptions{} + if deletion.uid != "" { + uid := types.UID(deletion.uid) + options.Preconditions = &metav1.Preconditions{UID: &uid} + } + err = deletion.call(ctx, deletion.name, options) + switch { + case err == nil, apierrors.IsNotFound(err): + default: + // A UID mismatch surfaces as a conflict. It means the name now + // belongs to a different object, which must not be deleted. + return fmt.Errorf("delete %s %s: %w", deletion.kind, deletion.name, err) + } + } + return nil +} + +// ResourcesAbsent reports whether every recorded object is gone, INCLUDING +// pods that are still terminating. Retirement completes only on verified +// absence; a Deployment that has been deleted while its pods still run is not +// an absent instance. +func (e *trinoPoolEffects) ResourcesAbsent(ctx context.Context, inventory trinoPoolInventory) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolRequestBudget) + defer cancel() + + for _, name := range []string{inventory.CoordinatorDeploymentName, inventory.WorkerDeploymentName} { + if name == "" { + continue + } + _, err := e.clientset.AppsV1().Deployments(inventory.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + if !apierrors.IsNotFound(err) { + return false, fmt.Errorf("observe deployment %s: %w", name, err) + } + } + if inventory.ServiceName != "" { + _, err := e.clientset.CoreV1().Services(inventory.Namespace).Get(ctx, inventory.ServiceName, metav1.GetOptions{}) + if err == nil { + return false, nil + } + if !apierrors.IsNotFound(err) { + return false, fmt.Errorf("observe service %s: %w", inventory.ServiceName, err) + } + } + pods, err := e.instancePods(ctx, inventory) + if err != nil { + return false, err + } + return len(pods) == 0, nil +} + +// Observe reports the cluster's current view of the instance. +func (e *trinoPoolEffects) Observe(ctx context.Context, inventory trinoPoolInventory) (trinoPoolObservation, error) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolRequestBudget) + defer cancel() + + var observation trinoPoolObservation + worker, err := e.clientset.AppsV1().Deployments(inventory.Namespace).Get(ctx, inventory.WorkerDeploymentName, metav1.GetOptions{}) + if err != nil { + return observation, fmt.Errorf("observe worker deployment: %w", err) + } + observation.ReadyWorkers = int(worker.Status.ReadyReplicas) + if worker.Spec.Replicas != nil { + observation.DesiredWorkers = int(*worker.Spec.Replicas) + } + + coordinator, err := e.clientset.AppsV1().Deployments(inventory.Namespace).Get(ctx, inventory.CoordinatorDeploymentName, metav1.GetOptions{}) + if err != nil { + return observation, fmt.Errorf("observe coordinator deployment: %w", err) + } + observation.CoordinatorReady = coordinator.Status.ReadyReplicas == 1 + + pods, err := e.instancePods(ctx, inventory) + if err != nil { + return observation, err + } + observation.PodsPresent = len(pods) + for _, pod := range pods { + if pod.Labels["app.kubernetes.io/component"] == componentCoordinatorLabel { + observation.CoordinatorPods = append(observation.CoordinatorPods, coordinatorPodStatus(pod)) + } + if pod.DeletionTimestamp != nil { + continue + } + switch pod.Labels["app.kubernetes.io/component"] { + case componentCoordinatorLabel: + observation.CoordinatorPodUID = string(pod.UID) + observation.CoordinatorImage = trinoContainerImage(pod) + case componentWorkerLabel: + // Any running worker: they are one Deployment, so a mixed set is + // itself a mid-rollout state the image comparison should catch. + if image := trinoContainerImage(pod); image != "" { + observation.WorkerImage = image + } + } + } + return observation, nil +} + +// coordinatorPodStatus projects the container-level facts a loss claim may rely +// on. Only the Trino container is read - a sidecar that restarted says nothing +// about the coordinator process - and a pod whose status the kubelet has not +// filled in yet simply reports no termination, which is not evidence. +func coordinatorPodStatus(pod corev1.Pod) trinoPoolCoordinatorPod { + status := trinoPoolCoordinatorPod{ + UID: string(pod.UID), + Terminating: pod.DeletionTimestamp != nil, + } + main := trinoContainerName(pod) + for _, container := range pod.Status.ContainerStatuses { + if container.Name != main { + continue + } + status.Restarts = container.RestartCount + status.RunningContainerID = container.ContainerID + if terminated := container.LastTerminationState.Terminated; terminated != nil { + status.LastTerminated = &trinoPoolContainerTermination{ + ContainerID: terminated.ContainerID, + ExitCode: terminated.ExitCode, + Reason: terminated.Reason, + FinishedAt: terminated.FinishedAt.UTC().Format(time.RFC3339), + } + } + } + return status +} + +// instancePods lists the instance's pods, terminating ones included: a pod that +// is going away still occupies the instance until it is actually gone. +func (e *trinoPoolEffects) instancePods(ctx context.Context, inventory trinoPoolInventory) ([]corev1.Pod, error) { + instance := instanceLabelFromInventory(inventory) + if instance == "" { + return nil, nil + } + pods, err := e.clientset.CoreV1().Pods(inventory.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: trinopool.LabelInstance + "=" + instance, + Limit: trinoPoolListLimit, + }) + if err != nil { + return nil, fmt.Errorf("observe instance pods: %w", err) + } + if pods.Continue != "" { + return nil, errors.New("instance pod inventory exceeds the listing limit") + } + return pods.Items, nil +} + +// instanceLabelFromInventory recovers the instance id from the recorded names. +// Every object is named "-" except the Service, which is named +// exactly after the instance. +func instanceLabelFromInventory(inventory trinoPoolInventory) string { + return inventory.ServiceName +} + +// Component label values, matching what Instantiate stamps. +const ( + componentCoordinatorLabel = "coordinator" + componentWorkerLabel = "worker" +) + +// trinoContainerImage reports the image of the pod's main Trino container. The +// name comes from the standard app label the chart already sets, so a sidecar's +// image can never be mistaken for the release image. +func trinoContainerImage(pod corev1.Pod) string { + if name := trinoContainerName(pod); name != "" { + for _, container := range pod.Spec.Containers { + if container.Name == name { + return container.Image + } + } + } + return "" +} + +// trinoContainerName is the name of the pod's main Trino container. The name +// comes from the standard app label the chart already sets, so a sidecar can +// never be mistaken for the coordinator. +func trinoContainerName(pod corev1.Pod) string { + main := pod.Labels["app.kubernetes.io/name"] + for _, container := range pod.Spec.Containers { + if main != "" && container.Name != main && !strings.HasPrefix(container.Name, "trino-") { + continue + } + if strings.HasPrefix(container.Name, "trino-") || container.Name == main { + return container.Name + } + } + return "" +} diff --git a/controlplane/trino_pool_effects_test.go b/controlplane/trino_pool_effects_test.go new file mode 100644 index 000000000..c08a4d6ac --- /dev/null +++ b/controlplane/trino_pool_effects_test.go @@ -0,0 +1,258 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/posthog/duckgres/controlplane/trinopool" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// testBlueprintJSON reads the same checked-in fixture the blueprint decoder +// tests use, so the effects layer is exercised against the document shape +// charts actually generates rather than a hand-made struct. +func testBlueprintJSON(t *testing.T) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("trinopool", "testdata", "blueprint.json")) + if err != nil { + t.Fatalf("read blueprint fixture: %v", err) + } + return data +} + +func testPoolObjects(t *testing.T, epoch int64) (*trinopool.Blueprint, trinopool.Identity, trinopool.Objects) { + t.Helper() + blueprint, err := trinopool.ParseBlueprint(testBlueprintJSON(t)) + if err != nil { + t.Fatalf("parse blueprint: %v", err) + } + identity := trinopool.Identity{ + PoolID: "registered:cell-001", PoolLabelValue: "cell-001", + InstanceID: "cell-001-a1b2c3d4", NodeEnvironment: "mw_dev_pool_001", + AuthorityEpoch: epoch, CoordinatorPort: 8443, + DiscoveryURIHost: "cell-001-a1b2c3d4.trino-pool-example.svc.cluster.local", + } + objects, err := blueprint.Instantiate(identity) + if err != nil { + t.Fatalf("instantiate: %v", err) + } + return blueprint, identity, objects +} + +// emulateUIDAssignment makes the fake clientset behave like a real API server +// for the one property this code depends on: every created object gets a UID. +// Without it the recorded inventory would be empty and every delete would lose +// its precondition, which is exactly the safety property under test. +func emulateUIDAssignment(clientset *fake.Clientset) { + var sequence int + clientset.PrependReactor("create", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + create, ok := action.(k8stesting.CreateAction) + if !ok { + return false, nil, nil + } + object, err := meta.Accessor(create.GetObject()) + if err != nil || object.GetUID() != "" { + return false, nil, nil + } + sequence++ + object.SetUID(types.UID(fmt.Sprintf("uid-%d", sequence))) + return false, nil, nil + }) +} + +func newEffects(t *testing.T, epoch int64) (*trinoPoolEffects, *fake.Clientset) { + t.Helper() + clientset := fake.NewClientset() + emulateUIDAssignment(clientset) + blueprint, _, _ := testPoolObjects(t, epoch) + return newTrinoPoolEffects(clientset, blueprint.Namespace, blueprint.SharedResources, epoch), clientset +} + +func TestEffectsApplyCreatesTheWholeInventory(t *testing.T) { + effects, clientset := newEffects(t, 7) + _, _, objects := testPoolObjects(t, 7) + + inventory, err := effects.Apply(context.Background(), objects) + if err != nil { + t.Fatalf("apply: %v", err) + } + if inventory.ServiceUID == "" || inventory.CoordinatorDeploymentUID == "" || inventory.WorkerDeploymentUID == "" { + t.Fatalf("inventory is missing recorded UIDs: %+v", inventory) + } + deployments, err := clientset.AppsV1().Deployments("trino-pool-example").List(context.Background(), metav1.ListOptions{}) + if err != nil || len(deployments.Items) != 2 { + t.Fatalf("deployments = %v (err %v)", deployments, err) + } +} + +// A lost create response must not produce a second instance. The names are +// deterministic and already recorded, so the next attempt reads the object back +// and adopts its own object. +func TestEffectsApplyIsIdempotent(t *testing.T) { + effects, clientset := newEffects(t, 7) + _, _, objects := testPoolObjects(t, 7) + + first, err := effects.Apply(context.Background(), objects) + if err != nil { + t.Fatalf("first apply: %v", err) + } + second, err := effects.Apply(context.Background(), objects) + if err != nil { + t.Fatalf("second apply: %v", err) + } + if first.CoordinatorDeploymentUID != second.CoordinatorDeploymentUID { + t.Fatal("a repeated apply replaced the coordinator deployment") + } + deployments, _ := clientset.AppsV1().Deployments("trino-pool-example").List(context.Background(), metav1.ListOptions{}) + if len(deployments.Items) != 2 { + t.Fatalf("a repeated apply created %d deployments", len(deployments.Items)) + } +} + +// An object that exists but belongs to something else is never adopted: doing +// so would let duckgres take over a workload another authority manages. +func TestEffectsRefuseToAdoptForeignObjects(t *testing.T) { + effects, clientset := newEffects(t, 7) + _, _, objects := testPoolObjects(t, 7) + + foreign := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: objects.Service.Name, + Namespace: objects.Service.Namespace, + Labels: map[string]string{"app.kubernetes.io/managed-by": "somebody-else"}, + }} + if _, err := clientset.CoreV1().Services(foreign.Namespace).Create(context.Background(), foreign, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed foreign service: %v", err) + } + if _, err := effects.Apply(context.Background(), objects); !errors.Is(err, errTrinoPoolForeignObject) { + t.Fatalf("apply error = %v, want errTrinoPoolForeignObject", err) + } +} + +// A superseded leader must not be able to mutate the current leader's objects. +func TestEffectsRefuseToWriteUnderAStaleEpoch(t *testing.T) { + current, clientset := newEffects(t, 9) + _, _, objects := testPoolObjects(t, 9) + if _, err := current.Apply(context.Background(), objects); err != nil { + t.Fatalf("apply: %v", err) + } + + blueprint, _, staleObjects := testPoolObjects(t, 4) + stale := newTrinoPoolEffects(clientset, blueprint.Namespace, blueprint.SharedResources, 4) + if _, err := stale.Apply(context.Background(), staleObjects); !errors.Is(err, errTrinoPoolStaleEpoch) { + t.Fatalf("stale apply error = %v, want errTrinoPoolStaleEpoch", err) + } + + service, err := clientset.CoreV1().Services("trino-pool-example").Get(context.Background(), objects.Service.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get service: %v", err) + } + if service.Annotations[trinopool.AnnotationAuthorityEpoch] != "9" { + t.Fatalf("stale leader lowered the epoch to %q", service.Annotations[trinopool.AnnotationAuthorityEpoch]) + } +} + +// Deleting pods while their controller still exists just makes new pods. Stop +// the controllers first, then verify absence. +func TestEffectsDeleteRemovesControllersAndRecordsAbsence(t *testing.T) { + effects, clientset := newEffects(t, 7) + _, _, objects := testPoolObjects(t, 7) + inventory, err := effects.Apply(context.Background(), objects) + if err != nil { + t.Fatalf("apply: %v", err) + } + + if err := effects.Delete(context.Background(), inventory); err != nil { + t.Fatalf("delete: %v", err) + } + deployments, _ := clientset.AppsV1().Deployments("trino-pool-example").List(context.Background(), metav1.ListOptions{}) + if len(deployments.Items) != 0 { + t.Fatalf("%d deployments survived deletion", len(deployments.Items)) + } + services, _ := clientset.CoreV1().Services("trino-pool-example").List(context.Background(), metav1.ListOptions{}) + if len(services.Items) != 0 { + t.Fatalf("%d services survived deletion", len(services.Items)) + } + + absent, err := effects.ResourcesAbsent(context.Background(), inventory) + if err != nil || !absent { + t.Fatalf("absent = %v (err %v)", absent, err) + } +} + +// Deleting an object whose UID moved on would destroy somebody else's +// replacement object that happens to share the name. +func TestEffectsDeleteIsUIDPreconditioned(t *testing.T) { + effects, clientset := newEffects(t, 7) + _, _, objects := testPoolObjects(t, 7) + inventory, err := effects.Apply(context.Background(), objects) + if err != nil { + t.Fatalf("apply: %v", err) + } + inventory.ServiceUID = "some-other-uid" + + if err := effects.Delete(context.Background(), inventory); err == nil { + t.Fatal("delete with a mismatched UID was accepted") + } + if _, err := clientset.CoreV1().Services("trino-pool-example").Get(context.Background(), objects.Service.Name, metav1.GetOptions{}); err != nil { + t.Fatalf("service was deleted despite the UID mismatch: %v", err) + } +} + +// Instance cleanup must never remove the pool's shared trust boundary. +func TestEffectsNeverDeleteSharedResources(t *testing.T) { + effects, clientset := newEffects(t, 7) + shared := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "trino-auth", Namespace: "trino-pool-example"}} + if _, err := clientset.CoreV1().Secrets(shared.Namespace).Create(context.Background(), shared, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed shared secret: %v", err) + } + + // A malformed inventory that names a shared object must be refused rather + // than executed. + if err := effects.Delete(context.Background(), trinoPoolInventory{ + Namespace: "trino-pool-example", ServiceName: "trino-auth", ServiceUID: "x", + }); !errors.Is(err, errTrinoPoolSharedResource) { + t.Fatalf("delete error = %v, want errTrinoPoolSharedResource", err) + } + if _, err := clientset.CoreV1().Secrets("trino-pool-example").Get(context.Background(), "trino-auth", metav1.GetOptions{}); err != nil { + t.Fatalf("shared secret was removed: %v", err) + } +} + +func TestEffectsObserveReportsWorkerReadiness(t *testing.T) { + effects, clientset := newEffects(t, 7) + _, _, objects := testPoolObjects(t, 7) + inventory, err := effects.Apply(context.Background(), objects) + if err != nil { + t.Fatalf("apply: %v", err) + } + + worker, err := clientset.AppsV1().Deployments(inventory.Namespace).Get(context.Background(), inventory.WorkerDeploymentName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get worker deployment: %v", err) + } + worker.Status = appsv1.DeploymentStatus{ReadyReplicas: 4, ObservedGeneration: worker.Generation} + if _, err := clientset.AppsV1().Deployments(inventory.Namespace).UpdateStatus(context.Background(), worker, metav1.UpdateOptions{}); err != nil { + t.Fatalf("update status: %v", err) + } + + observed, err := effects.Observe(context.Background(), inventory) + if err != nil { + t.Fatalf("observe: %v", err) + } + if observed.ReadyWorkers != 4 { + t.Fatalf("ready workers = %d, want 4", observed.ReadyWorkers) + } +} diff --git a/controlplane/trino_pool_failure.go b/controlplane/trino_pool_failure.go new file mode 100644 index 000000000..b7b3d956d --- /dev/null +++ b/controlplane/trino_pool_failure.go @@ -0,0 +1,536 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// The failure branch. +// +// A serving member whose pods are gone has NOT drained: its queries and +// transactions were lost, and recording that as a clean SEALED/RETIRED would +// erase the difference in the only durable record anyone will read afterwards. +// So failure is its own path, and every step of it is evidence-driven: +// +// - SUSPECT means "stop sending new work here". It authorizes nothing +// destructive and is reversible, because a probe failure is not death. +// - LOST requires POSITIVE evidence that the exact incarnation terminated. +// Kubernetes reporting the pods absent is such evidence; a timeout is not. +// - Only after the Gateway records the loss may the resources be removed, and +// the member is reported as failed rather than drained. + +// trinoPoolSuspectAfter is how long an admitted or serving instance may look +// unhealthy before it is excluded from new work. It is deliberately not a +// deletion timer: nothing is destroyed at the end of it. +const trinoPoolSuspectAfter = 2 * time.Minute + +// trinoPoolSuspectDrainAfter is how long a member may stay excluded before it +// is replaced through the PLANNED path. +// +// SUSPECT had exactly one exit that freed its slot: LOST, which requires +// verified absence of every recorded object. A crash-looping coordinator keeps +// its Deployment forever, so it kept its slot forever, and a second such +// failure exhausted the repair budget and stalled the pool. Draining is the +// honest alternative: it is refused if it would break the serving floor, it +// preserves whatever work the member still holds, and it ends in a retirement +// receipt rather than a loss claim nobody could prove. +const trinoPoolSuspectDrainAfter = 15 * time.Minute + +// trinoPoolIdentityObserveEvery paces the per-member identity probe. The +// question it answers - "is this still the process that was admitted" - changes +// only when a process restarts, so asking every tick would be one authenticated +// request per member per five seconds for an answer that almost never moves. +const trinoPoolIdentityObserveEvery = 30 * time.Second + +// observeHealth moves a serving or admitted instance onto the failure branch +// when the cluster stops reporting a healthy coordinator, or when the process +// behind it is no longer the incarnation that was admitted. +func (o *trinoPoolOperator) observeHealth(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + phase := trinopool.Phase(instance.Phase) + observed, err := o.kube(o.lease.Epoch).Observe(ctx, inventoryOf(instance)) + if err != nil { + // Not being able to LOOK is not evidence about the member. The instance + // keeps its phase and the next tick tries again. + return false, nil + } + + healthy := observed.CoordinatorReady && observed.CoordinatorPodUID == instance.CoordinatorPodUID + switch phase { + case trinopool.PhaseServing, trinopool.PhaseAdmitted: + if healthy { + // A ready pod with the same UID is not the same PROCESS: a + // container can restart inside it and come back ready with a new + // Trino incarnation. The Gateway binds the member to the boot + // identity it registered and refuses to dispatch to anything else, + // so this row would report SERVING while the pool quietly lost + // capacity - green here, empty there. + return o.observeProcessIdentity(ctx, instance, phase) + } + if time.Since(instance.PhaseChangedAt) < trinoPoolSuspectAfter { + return false, nil + } + return true, o.suspectInstance(ctx, instance, phase, "the coordinator is not reporting healthy") + case trinopool.PhaseSuspect: + // There is deliberately no path back to service. + // + // Suspicion is the GATEWAY's state as much as this row's: it excluded + // the member, and nothing short of a fresh certified admission puts it + // back. Flipping the local row to SERVING would leave the Gateway + // excluding a member this controller believes is serving - a row that + // says one thing while the pool does another. Re-admitting is worse: a + // member that failed its health check and recovered is an uncertain + // incarnation, and the pool has a cheap way to get a certain one. + // + // So a suspected member always leaves, and only the ROUTE depends on the + // evidence: proven dead, or drained like any planned replacement once + // capacity allows it. + if progressed, err := o.claimLossIfProven(ctx, instance, observed); progressed || err != nil { + return progressed, err + } + if time.Since(instance.PhaseChangedAt) < trinoPoolSuspectDrainAfter { + // A brief blip is given time to become provable one way or the + // other before its replacement is started. + return false, nil + } + return o.drainSuspectInstance(ctx, instance) + default: + return false, nil + } +} + +// observeProcessIdentity checks that the coordinator answering for a serving +// member is still the incarnation that was admitted. +// +// Bounded on purpose: it is one authenticated request per member per +// trinoPoolIdentityObserveEvery, not per tick, because the question it answers +// changes only when a process restarts. +// +// A probe that does not answer is NOT evidence. A timeout means the controller +// could not look - the same as a failed Observe - and a member is never +// suspected, let alone declared dead, on that basis. Only a DIFFERENT process +// identity is evidence, and the stored one is never quietly updated to match: +// the recorded identity is what the Gateway admitted, and a new process is a +// new member that has to earn its own admission. +func (o *trinoPoolOperator) observeProcessIdentity( + ctx context.Context, + instance configstore.TrinoPoolInstance, + phase trinopool.Phase, +) (bool, error) { + if o.identity == nil || instance.CoordinatorBootID == "" || instance.EndpointURL == "" { + return false, nil + } + if last, seen := o.identityObservedAt[instance.InstanceID]; seen && + time.Since(last) < trinoPoolIdentityObserveEvery { + return false, nil + } + + bootID, err := o.identity(ctx, instance.EndpointURL) + if err != nil { + slog.Debug("Trino pool member did not answer the identity probe.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "reason", err) + return false, nil + } + if o.identityObservedAt == nil { + o.identityObservedAt = map[string]time.Time{} + } + o.identityObservedAt[instance.InstanceID] = time.Now() + if bootID == instance.CoordinatorBootID { + return false, nil + } + + slog.Warn("Trino pool member is answering with a different process than the one admitted.", + "pool", o.config.PublicID, "instance", instance.InstanceID, + "admitted", instance.CoordinatorBootID, "observed", bootID) + return true, o.suspectInstance(ctx, instance, phase, + "the coordinator process restarted; the admitted incarnation is gone") +} + +func (o *trinoPoolOperator) suspectInstance(ctx context.Context, instance configstore.TrinoPoolInstance, from trinopool.Phase, reason string) error { + // The reason is chosen by whichever check fired first, and the Gateway + // hashes the whole request under the step identity, so a retry after a lost + // response that observes the OTHER condition would report a changed intent. + // Reading the member back resolves that without weakening the guard. + recorded, err := o.gateway.GetMember(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + // An unreadable member is an UNRESOLVED read-back, not permission to + // proceed on a body this controller derived meanwhile. + return fmt.Errorf("read back member %s: %w", instance.InstanceID, err) + } + if trinoPoolSuspicionRecorded[recorded.Phase] { + return o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + from, trinopool.PhaseSuspect, map[string]any{ + "gateway_state": recorded.Phase, + "gateway_generation": recorded.Generation, + "last_error": reason, + })) + } + member, err := o.gateway.SuspectMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.SuspectMemberRequest{ + Step: o.step(instance.InstanceID, "suspect"), + ExpectedGeneration: instance.GatewayGeneration, + Reason: reason, + }) + if err != nil { + return o.dropAuthority(fmt.Errorf("suspect member %s: %w", instance.InstanceID, err)) + } + slog.Warn("Trino pool instance suspected.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "reason", reason) + return o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + from, trinopool.PhaseSuspect, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + "last_error": reason, + })) +} + +// claimLossIfProven records a loss ONLY with positive evidence that the exact +// incarnation is gone. A failing probe is never that evidence, because a +// partitioned coordinator may still be serving queries nobody can see. +func (o *trinoPoolOperator) claimLossIfProven(ctx context.Context, instance configstore.TrinoPoolInstance, observed trinoPoolObservation) (bool, error) { + source, proven := o.processTerminationEvidence(ctx, instance, observed) + if !proven { + // Nothing proves the process ended: the member stays SUSPECT, excluded + // from new work, and nothing is deleted or declared. + return false, nil + } + + // A lost response is resolved by reading the member back - the claim below + // is byte-stable across attempts, but the Gateway may already have recorded + // it while the answer never arrived. + recorded, err := o.gateway.GetMember(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + return true, fmt.Errorf("read back member %s: %w", instance.InstanceID, err) + } + if trinoPoolLossAlreadyRecorded(recorded) { + return true, o.dropAuthority(o.recordLost(ctx, instance, recorded)) + } + member, err := o.gateway.LostMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.LostMemberRequest{ + Step: o.step(instance.InstanceID, "lost"), + ExpectedGeneration: instance.GatewayGeneration, + Evidence: trinogateway.EvidenceProcessTerminated, + Termination: trinogateway.TerminationProof{ + PodUID: instance.CoordinatorPodUID, + BootID: instance.CoordinatorBootID, + NodeID: instance.CoordinatorNodeID, + CoordinatorID: instance.CoordinatorID, + Source: source, + // ObservedAt is deliberately omitted. It is optional, the Gateway + // stamps its receipt itself, and the only values available here are + // either re-derived per attempt - which makes every retry after a + // lost response a changed intent under the same step identity - or + // not the termination-observation time at all. + }, + }) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("record loss of %s: %w", instance.InstanceID, err)) + } + return true, o.dropAuthority(o.recordLost(ctx, instance, member)) +} + +func (o *trinoPoolOperator) recordLost(ctx context.Context, instance configstore.TrinoPoolInstance, member trinogateway.Member) error { + slog.Warn("Trino pool instance lost; its work was not drained.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "retirementKind", member.RetirementKind) + return o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseSuspect, trinopool.PhaseLost, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + "failure_reason": "the coordinator process terminated with outstanding work", + }) +} + +// processTerminationEvidence looks for proof that the admitted coordinator +// process ENDED, and names what was seen so the Gateway's durable receipt says +// which observation it rests on. +// +// Two observations qualify, and both are statements Kubernetes makes only after +// the fact: +// +// - Every recorded object is verifiably gone. This is the pre-existing proof +// and the same standard the owned-resource teardown uses. +// - The container instance that hosted the admitted process is no longer +// running in the pod that hosted it. That is the case a coordinator which +// restarts IN PLACE produces: it keeps every object it had, so absence can +// never arrive, and the member was retained until the drain timer - where a +// drain cannot finish either, because the dead process's transactions stay +// pinned to it. +// +// The second one is only sound when it names the EXACT container instance that +// was admitted, which is why the id is recorded at registration. An earlier +// version accepted ANY termination record on the hosting pod plus a differing +// identity from the member's endpoint: a pod restarted once during startup +// carries such a record for its whole life, and a second coordinator pod makes +// the endpoint's answer ambiguous about which process replied - together they +// declared a live, serving member dead and wrote off its pinned work. +// +// Everything ambiguous fails closed: an unreadable cluster, an unenumerated pod +// list, more than one live coordinator pod, a member registered before the +// container id was recorded, a termination naming a different instance. Nothing +// here deletes or restarts anything to manufacture proof, and a timeout is +// never evidence. +// +// Evidence boundary, honestly: this rests on what the Kubernetes API reports. +// A force-deleted pod object, or a node partitioned from the API server, can +// leave a process running that the API no longer describes. Neither observation +// below can see that, so within that boundary "proven dead" means "Kubernetes +// reported it dead". +func (o *trinoPoolOperator) processTerminationEvidence( + ctx context.Context, + instance configstore.TrinoPoolInstance, + observed trinoPoolObservation, +) (string, bool) { + absent, err := o.kube(o.lease.Epoch).ResourcesAbsent(ctx, inventoryOf(instance)) + if err == nil && absent && observed.PodsPresent == 0 { + return "kubernetes-resources-absent", true + } + return o.admittedContainerEnded(instance, observed) +} + +func (o *trinoPoolOperator) admittedContainerEnded( + instance configstore.TrinoPoolInstance, + observed trinoPoolObservation, +) (string, bool) { + if instance.CoordinatorPodUID == "" || instance.CoordinatorContainerID == "" { + // Nothing was bound to a container instance, so no record can be + // correlated with the admitted process. Such a member leaves through + // the planned drain instead. + return "", false + } + live := 0 + var hosting *trinoPoolCoordinatorPod + for index, pod := range observed.CoordinatorPods { + if !pod.Terminating { + live++ + } + if pod.UID == instance.CoordinatorPodUID { + hosting = &observed.CoordinatorPods[index] + } + } + if hosting == nil || live > 1 { + // The hosting pod was not observed - which is "not seen", never + // "deleted", because this listing is label-filtered and labels are + // mutable - or more than one coordinator pod is live, which makes any + // statement about which process is which ambiguous. + return "", false + } + if hosting.RunningContainerID == instance.CoordinatorContainerID { + // The admitted container is still the running one. + return "", false + } + if hosting.LastTerminated == nil || hosting.RunningContainerID == "" { + // No termination has been recorded, or the kubelet has not reported a + // running container yet. Neither says the admitted instance ended. + return "", false + } + slog.Warn("Trino pool coordinator container ended; the admitted process is gone.", + "pool", o.config.PublicID, "instance", instance.InstanceID, + "admittedContainer", instance.CoordinatorContainerID, + "runningContainer", hosting.RunningContainerID, + "exitCode", hosting.LastTerminated.ExitCode, "reason", hosting.LastTerminated.Reason) + return "kubernetes-coordinator-container-terminated", true +} + +// Accepted read-back phases. The member lifecycle is a graph, not a ranking - +// SUSPECT can be reached from DRAINING and can itself go to DRAINING - so a +// read-back names the states that mean "this transition is already recorded" +// rather than inferring it from an order. +var ( + trinoPoolSuspicionRecorded = map[string]bool{"SUSPECT": true, "LOST": true} + trinoPoolLossRecorded = map[string]bool{"LOST": true} +) + +// trinoPoolLossAlreadyRecorded also accepts a retirement, but ONLY a failed one: +// a DRAINED retirement is the completion of a different transition and must +// never be read as the loss this claim is making. +func trinoPoolLossAlreadyRecorded(member trinogateway.Member) bool { + if trinoPoolLossRecorded[member.Phase] { + return true + } + return (member.Phase == "RETIRING" || member.Phase == "RETIRED") && member.RetirementKind == "FAILED" +} + +// cleanupFailedCandidate releases everything a candidate that can never be +// admitted is still holding. +// +// A FAILED_PREPARING instance is not finished business: its Deployments, +// Service and ConfigMaps are still running a whole Trino cluster, and its +// Gateway member is still PREPARING, which the Gateway counts as LIVE. One such +// instance at desired+surge is enough to refuse every later registration - no +// repair, no rollout - so this path is what keeps a single restarted candidate +// from wedging the pool. +// +// The route is the Gateway's OWN never-admitted retirement: a PREPARING member +// that has admitted no work may be retired directly, and the Gateway verifies +// that for itself rather than taking this controller's word for it. That claim +// is what authorizes deleting the objects, exactly as it does for a planned +// replacement - no loss claim, no termination evidence, and no pretending a +// candidate that never served was drained. +func (o *trinoPoolOperator) cleanupFailedCandidate(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + if instance.GatewayIncarnation == "" { + // The candidate failed before it ever registered, so there is no member + // to release: only the objects, if any, and the record. + return o.removeFailedCandidateResources(ctx, instance, trinogateway.Member{}) + } + + // The Gateway is authoritative for its own member, and every step bumps the + // generation, so the CAS value is read back rather than taken from the row. + member, err := o.gateway.GetMember(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + return true, fmt.Errorf("read failed candidate %s: %w", instance.InstanceID, err) + } + switch member.Phase { + case "RETIRING", "RETIRED": + return o.removeFailedCandidateResources(ctx, instance, member) + default: + retired, err := o.gateway.RetireMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "retire"), + ExpectedGeneration: member.Generation, + }) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("claim retirement of failed candidate %s: %w", instance.InstanceID, err)) + } + slog.Warn("Trino pool is retiring a candidate that could never be admitted.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "reason", failureReason(instance)) + return true, o.dropAuthority(o.store.RecordTrinoPoolInstanceFields(ctx, o.lease, instance.InstanceID, map[string]any{ + "gateway_state": retired.Phase, + "gateway_generation": retired.Generation, + })) + } +} + +// removeFailedCandidateResources deletes a retired candidate's objects and +// closes its record once they are verifiably gone. +func (o *trinoPoolOperator) removeFailedCandidateResources( + ctx context.Context, + instance configstore.TrinoPoolInstance, + member trinogateway.Member, +) (bool, error) { + inventory := inventoryOf(instance) + kube := o.kube(o.lease.Epoch) + if err := kube.Delete(ctx, inventory); err != nil { + return true, fmt.Errorf("clean up failed candidate %s: %w", instance.InstanceID, err) + } + absent, err := kube.ResourcesAbsent(ctx, inventory) + if err != nil || !absent { + // Deletion is in progress. The instance keeps its slot until absence is + // observed, so a terminating pod is never counted as freed capacity. + return false, err + } + updates := map[string]any{} + if member.InstanceID != "" && member.Phase != "RETIRED" { + reported, err := o.gateway.MemberRetired(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "retired"), + ExpectedGeneration: member.Generation, + ResourcesAbsent: true, + }) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("report retirement of failed candidate %s: %w", instance.InstanceID, err)) + } + updates["gateway_state"], updates["gateway_generation"] = reported.Phase, reported.Generation + } + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseFailedPreparing, trinopool.PhaseFailureRetired, updates)) +} + +// failureReason is what the Gateway records for the exclusion. It is never +// empty: the Gateway requires a reason, and "unknown" in a durable failure +// record is worse than a generic one. +func failureReason(instance configstore.TrinoPoolInstance) string { + if reason := instance.FailureReason; reason != "" { + return reason + } + return "the candidate could not be admitted" +} + +// drainSuspectInstance replaces a member that is neither healthy nor provably +// gone, through the ordinary drain. +// +// The Gateway decides whether it may go: a drain that would breach the serving +// floor is refused, and that refusal is authoritative - it is never overridden, +// so a pool that is already at its floor keeps the flaky member rather than +// dropping below it. Nothing is destroyed here either; the drain ends in a +// retirement claim like any planned replacement. +func (o *trinoPoolOperator) drainSuspectInstance(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + member, err := o.gateway.DrainMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "drain"), + ExpectedGeneration: instance.GatewayGeneration, + }) + if err != nil { + slog.Info("Trino pool cannot yet drain a suspected member.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "reason", err) + return false, o.dropAuthority(err) + } + slog.Warn("Trino pool is draining a member that stayed suspect.", + "pool", o.config.PublicID, "instance", instance.InstanceID, + "suspectFor", time.Since(instance.PhaseChangedAt).Round(time.Second)) + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseSuspect, trinopool.PhaseDraining, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) +} + +// completeFailureRetirement finishes a LOST member. The resources are already +// verifiably absent - that was the evidence for the loss claim - so this +// records the terminal state and releases the slot the planner's repair budget +// is waiting on. +func (o *trinoPoolOperator) completeFailureRetirement(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + inventory := inventoryOf(instance) + if err := o.kube(o.lease.Epoch).Delete(ctx, inventory); err != nil { + return true, fmt.Errorf("clean up lost instance %s: %w", instance.InstanceID, err) + } + absent, err := o.kube(o.lease.Epoch).ResourcesAbsent(ctx, inventory) + if err != nil || !absent { + return false, err + } + + // The Gateway's retirement protocol still has to run. Closing the local row + // while its member sat in LOST left the two records permanently + // disagreeing about whether that incarnation was finished with - and the + // retirement kind, FAILED, is the durable statement that its work was lost + // rather than drained. + member, err := o.gateway.GetMember(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + return true, fmt.Errorf("read lost member %s: %w", instance.InstanceID, err) + } + switch member.Phase { + case "RETIRED": + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseLost, trinopool.PhaseFailureRetired, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) + case "RETIRING": + reported, err := o.gateway.MemberRetired(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "retired"), + ExpectedGeneration: member.Generation, + ResourcesAbsent: true, + }) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("report retirement of lost member %s: %w", instance.InstanceID, err)) + } + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseLost, trinopool.PhaseFailureRetired, map[string]any{ + "gateway_state": reported.Phase, + "gateway_generation": reported.Generation, + })) + default: + claimed, err := o.gateway.RetireMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "retire"), + ExpectedGeneration: member.Generation, + }) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("claim retirement of lost member %s: %w", instance.InstanceID, err)) + } + return true, o.dropAuthority(o.store.RecordTrinoPoolInstanceFields(ctx, o.lease, instance.InstanceID, map[string]any{ + "gateway_state": claimed.Phase, + "gateway_generation": claimed.Generation, + })) + } +} diff --git a/controlplane/trino_pool_hoglake_wiring_test.go b/controlplane/trino_pool_hoglake_wiring_test.go new file mode 100644 index 000000000..1c9d01cc0 --- /dev/null +++ b/controlplane/trino_pool_hoglake_wiring_test.go @@ -0,0 +1,44 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "testing" + + "github.com/posthog/duckgres/controlplane/provisioner" + kubefake "k8s.io/client-go/kubernetes/fake" +) + +// A pooled cell gets the SAME storage inputs a legacy cell gets. +// +// Managed Hoglake needs two things the pool has no opinion about: the service +// configuration, and the resolver that reads the tenant's storage identity. +// They are wired once, for every cell, and a pooled cell that silently lost +// either would build, start and reconcile normally - right up to the first +// Hoglake tenant, which would then sit pending citing configuration nobody +// changed. +func TestPooledCellCarriesTheManagedHoglakeInputs(t *testing.T) { + t.Setenv(envTrinoFilesystemCacheEnabled, "false") + t.Setenv(envTrinoManagedHoglakeURI, "http://hoglake.example:8080") + t.Setenv(envTrinoHoglakeDataPath, "s3://example-bucket/trino/") + t.Setenv(envTrinoHoglakeNamespace, "") + + store := &fleetBootstrapStore{initialized: map[string]bool{}} + kc := kubefake.NewClientset() + ducklings := func(context.Context, string) (*provisioner.DucklingStatus, error) { return nil, nil } + storage := func(context.Context, string) (*provisioner.DucklingStatus, error) { return nil, nil } + + for _, cell := range []trinoCell{ + {ID: "cell-legacy", Namespace: "legacy", CoordinatorURL: "https://legacy.example.test"}, + {ID: registeredTrinoCellPrefix + "cell-pool", PublicID: "cell-pool", Namespace: "pooled", Mode: trinoPoolModeShared}, + } { + wire, err := buildTrinoCellWiring(store, kc, ducklings, cell, storage) + if err != nil { + t.Fatalf("wire cell %s: %v", cell.ID, err) + } + if !wire.Provisioner.ManagedHoglakeConfigured() { + t.Fatalf("cell %s cannot provision a managed Hoglake tenant: the service configuration or the storage resolver was dropped", cell.ID) + } + } +} diff --git a/controlplane/trino_pool_member_retries_test.go b/controlplane/trino_pool_member_retries_test.go new file mode 100644 index 000000000..d60f3c9dc --- /dev/null +++ b/controlplane/trino_pool_member_retries_test.go @@ -0,0 +1,295 @@ +//go:build kubernetes + +package controlplane + +// Member lifecycle retries under a lost response, and the failure repair of a +// coordinator that restarted in place. +// +// The Gateway journals every mutation under its step identity and hashes the +// WHOLE request body, minus the authority envelope. A retry that rebuilds any +// other field from freshly read state therefore reports a CHANGED INTENT, and +// the step can never be resolved again: not on the next tick, not after a +// leader change, never. These tests drive the real loop against the fake that +// journals payloads the same way. + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/posthog/duckgres/controlplane/trinogateway" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +func TestALostResponseDoesNotChangeTheRetriedMemberRequest(t *testing.T) { + // Registration re-observes the coordinator's boot identity on every + // attempt. If the response is lost and the coordinator restarts before the + // retry, the retry carries a different bootId under the same step id. + t.Run("register after an in-place restart", func(t *testing.T) { + harness := newOperatorHarness(t) + observations := 0 + harness.operator.identity = func(context.Context, string) (string, error) { + observations++ + if observations <= 1 { + return "process-1", nil + } + return "process-2", nil + } + harness.gateway.loseResponse = map[string]bool{"register": true} + + harness.tickTolerant(10) + + instanceID := harness.store.order[0] + instance := harness.store.instances[instanceID] + if instance.Phase == string(trinopool.PhaseCreating) { + t.Fatalf("phase = %s, want the lost registration resolved rather than a member "+ + "holding a live slot while the row never leaves CREATING", instance.Phase) + } + // The Gateway bound the member to the identity of the FIRST attempt, so + // that is the incarnation this row has to carry - never the new process. + if instance.CoordinatorBootID != "process-1" { + t.Fatalf("recorded boot id = %q, want the identity the Gateway recorded", instance.CoordinatorBootID) + } + }) + + // The suspicion reason is chosen by whichever check fired first. A lost + // response followed by a tick that observes the OTHER condition sends a + // different reason under the same step id. + t.Run("suspect with a different reason", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseServing, "ACTIVE") + + // First the restart check fires; its response is lost. + harness.operator.identity = func(context.Context, string) (string, error) { return "process-2", nil } + harness.gateway.loseResponse = map[string]bool{"suspect": true} + _ = harness.operator.reconcileOnce(context.Background()) + + // Then the coordinator stops reporting ready, so the next attempt would + // carry the unhealthy reason instead. + harness.kube.observed.CoordinatorReady = false + instance.PhaseChangedAt = time.Now().Add(-time.Hour) + harness.tickTolerant(5) + + if instance.Phase != string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want the lost suspicion resolved to SUSPECT", instance.Phase) + } + }) + + // The loss claim stamps a fresh observedAt on every attempt. + t.Run("lost", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseSuspect, "SUSPECT") + harness.kube.absent = true + harness.kube.observed.PodsPresent = 0 + harness.gateway.loseResponse = map[string]bool{"lost": true} + + // observedAt has second precision, so the retry must land in a later + // second - which is what every real reconcile tick does. + _ = harness.operator.reconcileOnce(context.Background()) + time.Sleep(1100 * time.Millisecond) + harness.tickTolerant(6) + + if instance.Phase == string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want the lost claim resolved: the member is LOST at the "+ + "Gateway and its objects are never cleaned up from here", instance.Phase) + } + }) +} + +// One instance that cannot make progress must not stop the pool: the planner +// runs after the instance loop, so an unrecoverable member used to block every +// repair, drain and replacement for as long as it stayed broken. +func TestOneUnrecoverableInstanceDoesNotStallTheRestOfThePool(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + + wedged := harness.store.order[0] + instance := harness.placeInstance(t, wedged, trinopool.PhaseSuspect, "SUSPECT") + // A claim the Gateway refuses outright: retrying cannot change a decision. + harness.gateway.lostErr = errors.New("gateway refused the loss claim") + harness.kube.absent = true + harness.kube.observed.PodsPresent = 0 + + harness.tickTolerant(10) + + if instance.Phase != string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want the refused claim to leave the member suspect", instance.Phase) + } + // The rest of the pool keeps converging: the planner still runs, so the + // member that cannot recover is replaced rather than holding the pool. + for _, id := range harness.store.order { + if id == wedged { + continue + } + if phase := harness.store.instances[id].Phase; phase == string(trinopool.PhasePending) { + t.Fatalf("instance %s = %s, want the other instances to keep progressing", id, phase) + } + } + if len(harness.store.order) < 4 { + t.Fatalf("instances = %d, want the planner to have started a replacement despite the "+ + "stuck member", len(harness.store.order)) + } +} + +// A coordinator that restarts in place keeps every object it had, so the +// absence evidence a loss claim used to require could never arrive. The member +// held its slot until the drain timer, and a drain cannot finish while the dead +// JVM's transactions are still pinned to it - so the instance, and the repair +// slot behind it, were retained forever. +func TestASamePodRestartWithPinnedWorkReachesFailureRepair(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + instanceID := harness.store.order[0] + instance := harness.store.instances[instanceID] + if instance.Phase != string(trinopool.PhaseServing) { + t.Fatalf("phase = %s, want a serving member to start from", instance.Phase) + } + + // Work the dead JVM will never finish: the Gateway keeps reporting it, + // because a lost process does not drain. + harness.gateway.obligations[instanceID] = trinogateway.Obligations{ + Generation: harness.gateway.members[instanceID].Generation, + Phase: "ACTIVE", OpenTransactions: 1, ActiveQueries: 2, + } + + // The container restarted inside the same Pod: same pod UID, a DIFFERENT + // container instance running, and Kubernetes' own record that a container + // ended. The admitted container id is what makes that record about THIS + // process rather than about some earlier restart. + harness.operator.identity = func(context.Context, string) (string, error) { return "process-2", nil } + harness.operator.identityObservedAt = nil + // Deleting the failed member's objects actually removes them, so the + // retirement this test is about can reach its end. + harness.kube.absentAfterDelete = true + harness.kube.observed.CoordinatorPods = []trinoPoolCoordinatorPod{{ + UID: "pod-uid-1", + Restarts: 1, + RunningContainerID: "containerd://restarted", + LastTerminated: &trinoPoolContainerTermination{ + ContainerID: "containerd://old", ExitCode: 137, Reason: "OOMKilled", + FinishedAt: "2026-09-19T12:00:00Z", + }, + }} + if got := harness.store.instances[instanceID].CoordinatorContainerID; got != "containerd://admitted" { + t.Fatalf("recorded container = %q, want the container the member was registered with", got) + } + + harness.tickTolerant(20) + + if instance.Phase != string(trinopool.PhaseFailureRetired) { + t.Fatalf("phase = %s, want the restarted member to complete failure repair", instance.Phase) + } + member := harness.gateway.members[instanceID] + if member.Phase != "RETIRED" || member.RetirementKind != "FAILED" { + t.Fatalf("gateway member = %s/%s, want RETIRED/FAILED - its work was lost, not drained", + member.Phase, member.RetirementKind) + } + // The claim rests on Kubernetes' own record that the container hosting the + // admitted process ended, bound to that exact incarnation. + claim := harness.gateway.lastLost + if claim.Evidence != trinogateway.EvidenceProcessTerminated || + claim.Termination.Source != "kubernetes-coordinator-container-terminated" { + t.Fatalf("loss evidence = %s/%s, want the container termination observation", + claim.Evidence, claim.Termination.Source) + } + if claim.Termination.BootID != "process-1" || claim.Termination.PodUID != "pod-uid-1" { + t.Fatalf("loss evidence identifies %s/%s, want the admitted incarnation", + claim.Termination.PodUID, claim.Termination.BootID) + } + // Its pinned work must never be reported as a clean drain. + for _, call := range harness.gateway.calls { + if call == "seal:"+instanceID { + t.Fatalf("sealed a member whose process died with work still pinned to it") + } + } + // The repair slot is released, so the pool can replace it. + if len(harness.store.order) < 4 { + t.Fatalf("instances = %d, want a replacement for the failed member", len(harness.store.order)) + } +} + +// A probe that answers with the admitted identity is not evidence of anything, +// and a pod that never restarted leaves no termination record. Neither may +// produce a loss claim: a member is only ever declared dead on proof. +func TestARestartlessSuspicionIsNeverDeclaredLost(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseSuspect, "SUSPECT") + + // The pod is present, has never restarted, and the process still answers + // with the identity that was admitted. + harness.kube.observed.CoordinatorPods = []trinoPoolCoordinatorPod{{UID: "pod-uid-1"}} + harness.operator.identity = func(context.Context, string) (string, error) { return "process-1", nil } + + harness.tickTolerant(5) + + if instance.Phase != string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want a suspicion with no termination evidence to stay suspect", instance.Phase) + } + for _, call := range harness.gateway.calls { + if call == "lost:"+instanceID { + t.Fatalf("claimed a loss without evidence that the process ended") + } + } +} + +// A termination record that belongs to some OTHER container instance is not +// evidence about the admitted one, and a second coordinator pod makes the +// endpoint's answer ambiguous about which process replied. +// +// The admitted pod carries a termination from BEFORE it was admitted - a +// restart during startup leaves exactly that - and is still serving its pinned +// work. A second coordinator pod overlaps it, as Kubernetes recovery routinely +// produces, and the endpoint answers as that pod's process. Nothing here says +// the admitted process ended, so nothing may declare it dead. +func TestALiveAdmittedProcessIsNotDeclaredLostByAnUncorrelatedTermination(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseSuspect, "SUSPECT") + harness.gateway.obligations[instanceID] = trinogateway.Obligations{ + Generation: harness.gateway.members[instanceID].Generation, + Phase: "SUSPECT", OpenTransactions: 1, + } + + harness.kube.observed.CoordinatorPods = []trinoPoolCoordinatorPod{ + { + // The admitted pod, still running the admitted container, carrying a + // termination record from a restart that happened before admission. + UID: "pod-uid-1", Restarts: 1, RunningContainerID: "containerd://admitted", + LastTerminated: &trinoPoolContainerTermination{ + ContainerID: "containerd://before-admission", ExitCode: 1, + FinishedAt: "2026-09-19T10:00:00Z", + }, + }, + // A second coordinator pod overlapping it. + {UID: "pod-uid-2", RunningContainerID: "containerd://other"}, + } + // The endpoint answers as the second pod's process. + harness.operator.identity = func(context.Context, string) (string, error) { return "process-2", nil } + harness.operator.identityObservedAt = nil + harness.kube.absentAfterDelete = true + + harness.tickTolerant(12) + + for _, call := range harness.gateway.calls { + if call == "lost:"+instanceID { + t.Fatalf("declared a live admitted process dead: its pinned work would be "+ + "written off on an uncorrelated termination record. calls = %v", harness.gateway.calls) + } + } + if instance.Phase == string(trinopool.PhaseLost) || + instance.Phase == string(trinopool.PhaseFailureRetired) { + t.Fatalf("phase = %s, want the member to stay suspect until it is actually proven dead", + instance.Phase) + } + if harness.kube.deleted[instance.ServiceName] { + t.Fatalf("deleted the objects of a member that was never proven dead") + } +} diff --git a/controlplane/trino_pool_operator.go b/controlplane/trino_pool_operator.go new file mode 100644 index 000000000..2e18add87 --- /dev/null +++ b/controlplane/trino_pool_operator.go @@ -0,0 +1,640 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// The shared-pool operator. +// +// It runs under the EXISTING janitor leader lease, but leadership only decides +// who executes. Correctness comes from the fences: the pool's authority epoch +// on every durable write, the same epoch on every Kubernetes object, and the +// Gateway's own epoch/generation CAS. A superseded leader that has not noticed +// yet is refused at each of those, not trusted to stop on its own. +// +// One step per instance per tick, and one lifecycle action per pool per tick. +// A stuck rollout must never be able to spawn a chain of replacements. +const ( + trinoPoolReconcileInterval = 5 * time.Second + trinoPoolInstanceIDBytes = 4 +) + +// trinoPoolStore is the durable state the operator needs. It is an interface so +// the loop can be tested without a database; the implementation's own semantics +// (fencing, CAS, replay) are covered by real-PostgreSQL tests. +type trinoPoolStore interface { + SeedTrinoPool(context.Context, configstore.TrinoPoolSpec) error + UpsertTrinoPoolSpec(context.Context, configstore.TrinoPoolLease, configstore.TrinoPoolSpec) error + FreezeTrinoPool(ctx context.Context, lease configstore.TrinoPoolLease, poolID, reason string) error + ThawTrinoPool(ctx context.Context, lease configstore.TrinoPoolLease, poolID string) error + GetTrinoPool(ctx context.Context, poolID string) (*configstore.TrinoPool, error) + AcquireTrinoPoolAuthority(ctx context.Context, poolID, owner string) (configstore.TrinoPoolLease, error) + ListTrinoPoolInstances(ctx context.Context, poolID string) ([]configstore.TrinoPoolInstance, error) + CreateTrinoPoolInstance(context.Context, configstore.TrinoPoolLease, configstore.TrinoPoolInstanceSpec) error + AdvanceTrinoPoolInstance(ctx context.Context, lease configstore.TrinoPoolLease, instanceID string, from, to trinopool.Phase, updates map[string]any) error + RecordTrinoPoolInstanceFields(ctx context.Context, lease configstore.TrinoPoolLease, instanceID string, updates map[string]any) error + // RecordTrinoPoolPublicationRevision checkpoints the published catalog + // revision the admission gate certifies members against. The operator needs + // it directly, not only through the catalog writer: a revision the writer + // committed but failed to record has to be recoverable by whoever next holds + // the authority. + RecordTrinoPoolPublicationRevision(ctx context.Context, lease configstore.TrinoPoolLease, poolID string, revision int64) error +} + +// trinoPoolGateway is the Gateway surface the operator uses. +type trinoPoolGateway interface { + EnsureInactiveBackend(context.Context, trinogateway.Backend) error + PublishTenantPrincipals(ctx context.Context, poolID, tenant string, request trinogateway.PublishPrincipalsRequest) (trinogateway.TenantAdmission, error) + ConfigurePool(context.Context, string, trinogateway.ConfigurePoolRequest) (trinogateway.PoolState, error) + GetPool(ctx context.Context, poolID string) (trinogateway.PoolState, error) + OpenPublication(ctx context.Context, poolID string, request trinogateway.OpenPublicationRequest) (trinogateway.Publication, error) + AbandonPublication(ctx context.Context, poolID, publicationID string, step trinogateway.Step) (trinogateway.Publication, error) + GetPublication(ctx context.Context, poolID, publicationID string) (trinogateway.Publication, error) + RecordPublicationReceipt(ctx context.Context, poolID, publicationID string, request trinogateway.PublicationReceiptRequest) (trinogateway.Publication, error) + CommitPublication(ctx context.Context, poolID, publicationID string, request trinogateway.CommitPublicationRequest) (trinogateway.Publication, error) + RevokeTenant(ctx context.Context, poolID, tenant string, request trinogateway.RevokeTenantRequest) (trinogateway.TenantAdmission, error) + RegisterMember(context.Context, string, trinogateway.RegisterMemberRequest) (trinogateway.Member, error) + AdmitMember(ctx context.Context, poolID, instanceID string, request trinogateway.AdmitMemberRequest) (trinogateway.Member, error) + GetMember(ctx context.Context, poolID, instanceID string) (trinogateway.Member, error) + GetObligations(ctx context.Context, poolID, instanceID string) (trinogateway.Obligations, error) + DrainMember(ctx context.Context, poolID, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) + SealMember(ctx context.Context, poolID, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) + SuspectMember(ctx context.Context, poolID, instanceID string, request trinogateway.SuspectMemberRequest) (trinogateway.Member, error) + LostMember(ctx context.Context, poolID, instanceID string, request trinogateway.LostMemberRequest) (trinogateway.Member, error) + RetireMember(ctx context.Context, poolID, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) + MemberRetired(ctx context.Context, poolID, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) +} + +// trinoPoolKube is the Kubernetes surface the operator uses. +type trinoPoolKube interface { + Apply(context.Context, trinopool.Objects) (trinoPoolInventory, error) + Observe(context.Context, trinoPoolInventory) (trinoPoolObservation, error) + Delete(context.Context, trinoPoolInventory) error + ResourcesAbsent(context.Context, trinoPoolInventory) (bool, error) +} + +// trinoPoolValidator probes a candidate through its own endpoint. +type trinoPoolValidator func(ctx context.Context, endpoint string, observed trinoPoolObservation, expected trinoPoolExpectation) (trinoPoolValidation, error) + +// trinoPoolIdentityProbe reads a candidate's coordinator process identity. +type trinoPoolIdentityProbe func(ctx context.Context, endpoint string) (string, error) + +type trinoPoolOperator struct { + config trinoPoolConfig + store trinoPoolStore + gateway trinoPoolGateway + kube func(epoch int64) trinoPoolKube + validate trinoPoolValidator + identity trinoPoolIdentityProbe + // projection reports what this control plane currently serves: the + // authorization bundle's revision and the fingerprints of the projected + // password and group files. It is what a member is compared against when + // its acknowledgement is recorded. + projection func() trinoPoolProjectionRevisions + // acceptedProjection reports the projection the pool's DURABLE record + // accepts. Candidate admission compares against this rather than against + // what this process last published, because a replica's own memory is + // exactly the thing in question when replicas disagree. + acceptedProjection func() string + // resolveConfig re-reads this pool's desired configuration from the + // authoritative source. It runs on every tick, immediately before the + // desired state is published, so a process that has been idle since boot + // cannot publish what it read then. Nil in tests that drive a fixed config. + resolveConfig func() (trinoPoolConfig, error) + owner string + interval time.Duration + // operatorEnabled gates every external effect. With it off the operator + // keeps the durable desired state in sync and touches nothing else, which + // is how the feature ships disabled without the code path rotting. + operatorEnabled bool + newInstanceID func() string + // installWriter claims the catalog store's writer fence under the lease + // just acquired and installs it as the cell's catalog write path. + installWriter func(context.Context, configstore.TrinoPoolLease) error + // releaseWriter drops the authority the catalog writer publishes under. It + // runs when the leadership term ends, so a superseded process stops + // attempting writes rather than discovering the fence one publication at a + // time. + releaseWriter func() + // operations records durable intents around external effects, so a lost + // response is resolved by read-back rather than repeated blind. + operations trinoPoolOperationStore + // tenants is the org projection the principal binding is derived from. + tenants trinoPoolTenantStore + // publications is the durable record of which tenant is published and + // admitted at which revision. It is durable rather than remembered because + // a restart or a leadership move must not republish blindly, nor assume an + // admission that never committed. + publications trinoPoolPublicationStore + // acknowledgement asks ONE member what configuration it is serving, which + // is what a publication receipt asserts. + acknowledgement func(ctx context.Context, endpoint string, expected trinoPoolProjectionRevisions, catalogRevision int64) (trinoPoolAcknowledgement, error) + // catalogWatermark reports the revision the CATALOG STORE is at, which is + // the authority for the admission gate. The pool row's publication_revision + // is a cache of it, and a cache whose write failed is indistinguishable from + // "nothing new was published" unless the store is asked. Nil for a cell that + // publishes through a coordinator, where the store is not duckgres-side and + // this question has no local answer. + catalogWatermark func(ctx context.Context) (int64, error) + // bindingCursor and barrierCursor rotate which tenant is worked on. The + // driver performs one external step per tick, so a fixed order lets one + // permanently failing tenant hold the front of the queue forever - and with + // thousands of warehouses, "forever" is not hyperbole. + bindingCursor uint64 + barrierCursor uint64 + // identityObservedAt paces the per-member process-identity probe. It is + // memory rather than durable state because it only spaces out a question + // whose answer is re-read anyway; a restart simply asks again. + identityObservedAt map[string]time.Time + // tenantTurn alternates the two long queues - publishing a changed binding + // and opening the next barrier - so neither can starve the other at fleet + // scale. + tenantTurn uint64 + // barrierBasis is the configuration the live publication attempt was opened + // against, so every receipt it collects attests to ONE configuration rather + // than to whatever was current when each was taken. At most one attempt is + // live, so this holds at most one entry. It is deliberately per-process: an + // attempt this process did not open is released and reopened rather than + // completed on evidence nobody can describe. + barrierBasis map[string]trinoPoolBarrierBasis + + lease configstore.TrinoPoolLease + // fenced records that this term lost the fence. It ends the loop rather + // than letting a superseded controller re-acquire. + fenced bool + // pool is the durable row read at the start of the tick, so the steps agree + // on one view of the desired state. + pool *configstore.TrinoPool +} + +// Run is the leader-attached loop. It is started fresh on every leadership +// acquisition and cancelled on loss, so it re-acquires authority each time +// rather than trusting a lease it held before. +func (o *trinoPoolOperator) Run(ctx context.Context) { + interval := o.interval + if interval <= 0 { + interval = trinoPoolReconcileInterval + } + o.fenced = false + + // A new Run is a NEW leadership term. Any lease left on the struct belongs + // to the previous term and must not be reused: the janitor lease may have + // moved away and back, and another control plane may have taken the pool's + // authority in between. + o.lease = configstore.TrinoPoolLease{} + if o.releaseWriter != nil { + // The catalog writer publishes under THIS term's authority, so it has + // none until the term acquires one, and none again once it ends. + o.releaseWriter() + defer o.releaseWriter() + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + if err := o.reconcileOnce(ctx); err != nil && ctx.Err() == nil { + if errors.Is(err, errTrinoPoolBackoff) { + // Nothing was attempted: an operation is serving out the wait a + // previous failure earned. That is the retry schedule working, + // not a fault to alert on. + slog.Debug("Trino pool operation is waiting for its next attempt.", + "pool", o.config.PublicID, "reason", err) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + continue + } + slog.Warn("Trino pool reconcile failed.", "pool", o.config.PublicID, "error", err) + if o.fenced { + // The fence refused this leader. Ending the term is the correct + // response: re-acquiring here would ratchet the epoch against a + // valid new leader on every tick, and two controllers taking + // turns raising the epoch is worse than one stepping aside. The + // janitor lease decides when this process leads again. + slog.Warn("Trino pool leadership term ended after a fence refusal.", + "pool", o.config.PublicID, "epoch", o.lease.Epoch) + return + } + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (o *trinoPoolOperator) reconcileOnce(ctx context.Context) error { + // A pool whose desired configuration is unreadable freezes at its last-good + // state. No creates, no drains, no deletes - and explicitly not a desired + // count of zero. + // Desired-state publication is a lifecycle-affecting write, so it is fenced + // like every other one. Without the lease, a delayed old leader - or a + // replica still holding stale configuration - could overwrite a newer + // desired spec or clear another leader's freeze. A read-only operator does + // not write it at all: it has no authority to speak for the pool. + if !o.operatorEnabled { + return nil + } + // Desired configuration is re-read from the authoritative source on every + // tick, not taken from a snapshot made when this process booted. + // + // The startup snapshot was the actual staleness hazard. A replica that + // booted before a configuration change, sat idle, and then won the janitor + // lease would publish what it read at boot - a perfectly legal fenced write + // of old content, and one the generation guard cannot catch, because the + // value that orders generations does not change for a settings-only edit + // and two different configurations can legitimately carry the same one. + // Reading immediately before writing makes the published spec current by + // construction, and bounds the window to a single tick. + o.refreshConfig() + // Seeding is the one unfenced write, and it can only INSERT: a fence needs + // a row to lock, so the very first publication has to create one. + if err := o.store.SeedTrinoPool(ctx, o.config.Spec); err != nil { + return fmt.Errorf("seed pool row: %w", err) + } + if err := o.ensureAuthority(ctx); err != nil { + return err + } + if o.config.Frozen { + return o.dropAuthority(o.store.FreezeTrinoPool(ctx, o.lease, o.config.PoolID, o.config.FrozenReason)) + } + if err := o.store.UpsertTrinoPoolSpec(ctx, o.lease, o.config.Spec); err != nil { + if errors.Is(err, configstore.ErrTrinoPoolStaleGeneration) { + // The configuration this leader holds carries a generation the + // store has already passed. That is a configuration problem, not a + // lost fence: ending the term would hand the pool to a replica + // reading the same file and doing the same thing. Hold the last-good + // state and say why, which is what every other unusable desired + // configuration does. + slog.Error("Trino pool desired configuration went backwards; holding the last-good state.", + "pool", o.config.PublicID, "error", err) + return o.dropAuthority(o.store.FreezeTrinoPool(ctx, o.lease, o.config.PoolID, + "desired configuration is older than the published one: "+err.Error())) + } + return o.dropAuthority(fmt.Errorf("record desired pool spec: %w", err)) + } + if err := o.store.ThawTrinoPool(ctx, o.lease, o.config.PoolID); err != nil { + return o.dropAuthority(fmt.Errorf("clear pool freeze: %w", err)) + } + + pool, err := o.store.GetTrinoPool(ctx, o.config.PoolID) + if err != nil { + return fmt.Errorf("read pool state: %w", err) + } + if pool == nil { + return fmt.Errorf("pool %s has no durable row", o.config.PoolID) + } + o.pool = pool + if err := o.configureGatewayPool(ctx); err != nil { + return err + } + // The binding has to be current BEFORE the gate can refuse anything on its + // basis, and a tenant is only dispatchable once its barrier has committed, + // so both run before any lifecycle step. + // + // A tenant's failure is NOT allowed to stop the pool. One warehouse whose + // publication can never succeed used to end the tick here, so no instance + // was repaired, drained or replaced for as long as it stayed broken - the + // pool's compute lifecycle held hostage by one row. The failure is recorded + // against that tenant (with its own backoff) and reported at the end; only + // a lost fence stops the tick, because after that nothing this process + // writes can land anyway. + tenantErr := o.advanceTenantAdmissions(ctx) + if tenantErr != nil { + if o.fenced { + return tenantErr + } + slog.Warn("Trino pool tenant admission step failed; continuing with the instance lifecycle.", + "pool", o.config.PublicID, "error", tenantErr) + } + + instances, err := o.store.ListTrinoPoolInstances(ctx, o.config.PoolID) + if err != nil { + return fmt.Errorf("list pool instances: %w", err) + } + // Advance the instances already in flight before starting anything new, so + // a slow rollout cannot be overtaken by its own successor. + // + // An instance failure is reported but does not end the tick: the planner is + // what repairs and replaces members, and holding it back because one member + // is stuck is how a single unrecoverable instance took the whole pool's + // lifecycle with it. + progressed, instanceErr := o.progressInstances(ctx, instances) + if instanceErr != nil && o.fenced { + return errors.Join(tenantErr, instanceErr) + } + if progressed { + return errors.Join(tenantErr, instanceErr) + } + return errors.Join(tenantErr, instanceErr, o.applyPlan(ctx, pool, instances)) +} + +// refreshConfig replaces the desired configuration with what the authoritative +// source says NOW. +// +// An unreadable source freezes the pool at its last-good state rather than +// failing the tick: the pool keeps serving, nothing is created, drained or +// deleted, and the operator is told why. It is explicitly NOT a desired count +// of zero and explicitly NOT a startup abort - a bad mount must not empty a +// fleet or take the control plane down. +// +// A resolution that names a DIFFERENT pool is refused outright: it would mean +// this loop is about to publish another pool's desired state under this pool's +// authority. +func (o *trinoPoolOperator) refreshConfig() { + if o.resolveConfig == nil { + return + } + fresh, err := o.resolveConfig() + if err != nil { + o.config.Frozen = true + o.config.FrozenReason = "desired configuration is unreadable: " + err.Error() + slog.Warn("Trino pool desired configuration is unreadable; holding the last-good state.", + "pool", o.config.PublicID, "error", err) + return + } + if fresh.PoolID != o.config.PoolID { + o.config.Frozen = true + o.config.FrozenReason = fmt.Sprintf("desired configuration now names pool %q", fresh.PoolID) + slog.Error("Trino pool desired configuration names a different pool; holding the last-good state.", + "pool", o.config.PublicID, "resolved", fresh.PoolID) + return + } + o.config = fresh +} + +// ensureAuthority acquires the pool's authority epoch once per leadership term. +// Losing it is not something to retry harder: the next tick re-acquires, and +// until then every fenced write correctly refuses. +func (o *trinoPoolOperator) ensureAuthority(ctx context.Context) error { + if o.lease.Epoch != 0 { + return nil + } + lease, err := o.store.AcquireTrinoPoolAuthority(ctx, o.config.PoolID, o.owner) + if err != nil { + return fmt.Errorf("acquire pool authority: %w", err) + } + o.lease = lease + slog.Info("Trino pool authority acquired.", "pool", o.config.PublicID, "epoch", lease.Epoch) + + // The catalog writer's fence IS this authority: claim it now, under the + // epoch we just won, and install it as the cell's write path. Claiming at + // startup instead would have every replica take the cell on boot, which + // would make the writer fence agree with everyone and distinguish nobody. + if o.installWriter != nil { + if err := o.installWriter(ctx, lease); err != nil { + // Authority is held but catalogs cannot be published. Publishing + // through a stale path would be worse, so the pool keeps serving and + // the failure is surfaced for the next tick to retry. + return fmt.Errorf("claim catalog writer for pool %s: %w", o.config.PublicID, err) + } + } + return nil +} + +// dropAuthority is called when a fenced write is refused. The leader has been +// superseded, so it marks the term finished and stops. It does NOT re-acquire: +// a stale controller that immediately bumps the epoch again would fence the +// valid leader right back, and the two would trade the pool forever. Ending the +// term hands the decision back to the janitor lease, which is the only thing +// that knows who should be leading. +func (o *trinoPoolOperator) dropAuthority(err error) error { + if errors.Is(err, configstore.ErrTrinoPoolConflict) || errors.Is(err, trinogateway.ErrStaleEpoch) { + slog.Warn("Trino pool authority lost.", "pool", o.config.PublicID, "epoch", o.lease.Epoch, "error", err) + o.lease = configstore.TrinoPoolLease{} + o.fenced = true + if o.releaseWriter != nil { + o.releaseWriter() + } + } + return err +} + +func (o *trinoPoolOperator) configureGatewayPool(ctx context.Context) error { + _, err := o.gateway.ConfigurePool(ctx, o.config.RoutingGroup, trinogateway.ConfigurePoolRequest{ + Step: trinogateway.Step{ + OperationID: "pool-config:" + o.config.PublicID, + // The step identity carries BOTH the desired shape and this + // leader's epoch. + // + // The Gateway hashes the whole request body, epoch included, so a + // new leader re-sending an unchanged configuration under the same + // step id would hash differently and conflict forever. Putting the + // epoch in the step id makes each leadership term its own step: + // a repeat within one term is still a replay, and a new term is a + // new step rather than a permanent conflict. + StepID: fmt.Sprintf("configure.e%d.%s", o.lease.Epoch, o.configDigest()), + ControllerEpoch: o.lease.Epoch, + // The owner is what makes an EQUAL epoch from a different process + // refusable. Without it the Gateway's recorded owner stays NULL and + // the epoch alone fences, which admits a second controller at the + // same epoch. + OwnerIdentity: o.owner, + }, + APIMode: "POOLED", + MinServing: o.config.Spec.MinServing, + DesiredMembers: o.config.Spec.DesiredInstances, + MaxSurge: o.config.Spec.MaxSurge, + MaxRepair: o.config.Spec.MaxRepair, + DesiredRevision: o.config.Spec.DesiredReleaseID, + // The gate is a deliberate per-pool choice. It is deny-only: with it on, + // a tenant whose principals this controller has not published yet + // cannot dispatch work. Publishing the binding is therefore part of the + // same loop (see publishTenantBindings). + TenantAdmissionEnabled: o.config.Pool.TenantAdmission, + }) + if err != nil { + return o.dropAuthority(fmt.Errorf("configure gateway pool: %w", err)) + } + return nil +} + +func (o *trinoPoolOperator) configDigest() string { + digest := o.config.Spec.DesiredBlueprintDigest + if len(digest) > 8 { + digest = digest[:8] + } + if digest == "" { + digest = "none" + } + return fmt.Sprintf("%s-%d-%d-%d-%d", digest, + o.config.Spec.DesiredInstances, o.config.Spec.MinServing, o.config.Spec.MaxSurge, o.config.Spec.MaxRepair) +} + +// applyPlan starts at most one lifecycle action. +func (o *trinoPoolOperator) applyPlan(ctx context.Context, pool *configstore.TrinoPool, instances []configstore.TrinoPoolInstance) error { + views := make([]trinopool.InstanceView, 0, len(instances)) + for _, instance := range instances { + views = append(views, instance.View()) + } + plan := trinopool.PlanNext(trinopool.PoolState{ + DesiredInstances: pool.DesiredInstances, + MinServing: pool.MinServing, + MaxSurge: pool.MaxSurge, + MaxRepair: pool.MaxRepair, + DesiredReleaseID: pool.DesiredReleaseID, + Frozen: pool.Frozen, + FrozenReason: pool.FrozenReason, + Instances: views, + }) + switch plan.Action { + case trinopool.PlanActionCreate: + return o.createInstance(ctx, plan) + case trinopool.PlanActionDrain: + return o.beginDrain(ctx, plan) + default: + return nil + } +} + +// createInstance persists the identity BEFORE anything exists in Kubernetes. +// That ordering is what makes a lost create response recoverable: the name is +// deterministic and already recorded, so the next tick reads it back instead of +// creating a second instance. +func (o *trinoPoolOperator) createInstance(ctx context.Context, plan trinopool.Plan) error { + suffix := o.newInstanceID() + if suffix == "" { + // A random suffix is what keeps instance identities from being reused. + // Without one, this create would mint "-" and collide with itself + // on the next attempt. + return errors.New("could not generate an instance identity") + } + instanceID := o.config.PublicID + "-" + suffix + identity := o.identityFor(instanceID) + objects, err := o.config.Blueprint.Instantiate(identity) + if err != nil { + return fmt.Errorf("instantiate instance %s: %w", instanceID, err) + } + spec := configstore.TrinoPoolInstanceSpec{ + InstanceID: instanceID, + RepairFor: plan.RepairFor, + PoolID: o.config.PoolID, + ReleaseID: o.config.Blueprint.ReleaseID, + SpecDigest: o.config.Blueprint.SpecDigest(identity), + BlueprintSnapshot: o.blueprintSnapshot(), + Phase: trinopool.PhasePending, + Repair: plan.Repair, + EndpointURL: o.endpointFor(instanceID), + } + if err := o.store.CreateTrinoPoolInstance(ctx, o.lease, spec); err != nil { + return o.dropAuthority(fmt.Errorf("record instance %s: %w", instanceID, err)) + } + slog.Info("Trino pool instance created.", "pool", o.config.PublicID, "instance", instanceID, + "repair", plan.Repair, "repairFor", plan.RepairFor, "reason", plan.Reason) + // objects is discarded on purpose: this call is a pre-flight that the + // identity CAN be instantiated before the row is written. The objects + // themselves are created on the next tick, from the instance's own stored + // snapshot rather than from live configuration. + _ = objects + return nil +} + +func (o *trinoPoolOperator) beginDrain(ctx context.Context, plan trinopool.Plan) error { + instance, err := o.instanceByID(ctx, plan.InstanceID) + if err != nil { + return err + } + member, err := o.gateway.DrainMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "drain"), + ExpectedGeneration: instance.GatewayGeneration, + }) + if err != nil { + // A serving-floor refusal is the Gateway doing its job. It is recorded + // and retried on a later tick, never overridden. + return o.dropAuthority(fmt.Errorf("drain member %s: %w", instance.InstanceID, err)) + } + return o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseServing, trinopool.PhaseDraining, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) +} + +func (o *trinoPoolOperator) instanceByID(ctx context.Context, instanceID string) (configstore.TrinoPoolInstance, error) { + instances, err := o.store.ListTrinoPoolInstances(ctx, o.config.PoolID) + if err != nil { + return configstore.TrinoPoolInstance{}, err + } + for _, instance := range instances { + if instance.InstanceID == instanceID { + return instance, nil + } + } + return configstore.TrinoPoolInstance{}, fmt.Errorf("instance %s is unknown", instanceID) +} + +func (o *trinoPoolOperator) identityFor(instanceID string) trinopool.Identity { + return trinopool.Identity{ + PoolID: o.config.PoolID, + PoolLabelValue: o.config.PublicID, + InstanceID: instanceID, + NodeEnvironment: o.config.Pool.NodeEnvironment, + AuthorityEpoch: o.lease.Epoch, + CoordinatorPort: o.config.Pool.CoordinatorServicePort, + DiscoveryURIHost: o.serviceHost(instanceID), + } +} + +func (o *trinoPoolOperator) serviceHost(instanceID string) string { + return fmt.Sprintf("%s.%s.svc.cluster.local", instanceID, o.config.Namespace) +} + +// endpointFor is the instance's own in-cluster Service, over plain HTTP. +// +// TLS terminates at the Gateway; there is no per-instance certificate, no +// private CA and no trust distribution. Everything that talks to a coordinator +// this way declares the forwarded HTTPS hop instead of relaxing +// authentication. The tradeoff is explicit: credentials and query data cross +// the cluster network unencrypted between Gateway/operator and coordinator. +func (o *trinoPoolOperator) endpointFor(instanceID string) string { + return fmt.Sprintf("http://%s:%d", o.serviceHost(instanceID), o.config.Pool.CoordinatorServicePort) +} + +func (o *trinoPoolOperator) backendName(instanceID string) string { + return o.config.RoutingGroup + "-" + instanceID +} + +// step builds the idempotency envelope. The operation id is the instance's, so +// every step of one instance's lifecycle is resolvable as a single history. +func (o *trinoPoolOperator) step(instanceID, stepID string) trinogateway.Step { + return trinogateway.Step{ + OperationID: "instance:" + instanceID, + StepID: stepID, + ControllerEpoch: o.lease.Epoch, + OwnerIdentity: o.owner, + } +} + +func (o *trinoPoolOperator) blueprintSnapshot() string { + encoded, err := o.config.Blueprint.MarshalSnapshot() + if err != nil { + return "{}" + } + return encoded +} + +func newTrinoPoolInstanceID() string { + buffer := make([]byte, trinoPoolInstanceIDBytes) + if _, err := rand.Read(buffer); err != nil { + // A random suffix is what keeps instance identities from being reused. + // Falling back to something predictable would break that, so fail the + // creation instead and let the next tick try again. + return "" + } + return hex.EncodeToString(buffer) +} diff --git a/controlplane/trino_pool_operator_test.go b/controlplane/trino_pool_operator_test.go new file mode 100644 index 000000000..f62f5502b --- /dev/null +++ b/controlplane/trino_pool_operator_test.go @@ -0,0 +1,3616 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "testing" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// --------------------------------------------------------------------------- +// Fakes. The store's own semantics (fencing, CAS, replay) are covered by +// real-PostgreSQL tests; these fakes exist so the LOOP's decisions can be +// asserted without a database. +// --------------------------------------------------------------------------- + +type fakePoolStore struct { + pool *configstore.TrinoPool + instances map[string]*configstore.TrinoPoolInstance + order []string + epoch int64 + frozen string + // failAdvance simulates losing authority mid-tick. + failAdvance bool + // staleGeneration simulates a desired spec whose generation is behind the + // published one. + staleGeneration bool + // failRevisionCheckpoint simulates the write of the published catalog + // revision failing after the catalog itself committed. + failRevisionCheckpoint bool + // recordedRevisions is every checkpoint the operator wrote, in order. + recordedRevisions []int64 +} + +func (f *fakePoolStore) RecordTrinoPoolPublicationRevision(_ context.Context, lease configstore.TrinoPoolLease, _ string, revision int64) error { + if lease.Epoch != f.epoch { + return configstore.ErrTrinoPoolConflict + } + if f.failRevisionCheckpoint { + return errors.New("checkpoint refused") + } + f.recordedRevisions = append(f.recordedRevisions, revision) + f.pool.PublicationRevision = revision + return nil +} + +func newFakePoolStore(spec configstore.TrinoPoolSpec) *fakePoolStore { + return &fakePoolStore{ + pool: &configstore.TrinoPool{ + PoolID: spec.PoolID, PublicID: spec.PublicID, APIMode: spec.APIMode, + DesiredInstances: spec.DesiredInstances, MinServing: spec.MinServing, + MaxSurge: spec.MaxSurge, MaxRepair: spec.MaxRepair, + DesiredReleaseID: spec.DesiredReleaseID, + }, + instances: map[string]*configstore.TrinoPoolInstance{}, + } +} + +func (f *fakePoolStore) SeedTrinoPool(_ context.Context, _ configstore.TrinoPoolSpec) error { + return nil +} + +func (f *fakePoolStore) UpsertTrinoPoolSpec(_ context.Context, lease configstore.TrinoPoolLease, spec configstore.TrinoPoolSpec) error { + if lease.Epoch != f.epoch { + return configstore.ErrTrinoPoolConflict + } + if f.staleGeneration { + return fmt.Errorf("%w: 1 is behind the published 2", configstore.ErrTrinoPoolStaleGeneration) + } + f.pool.DesiredInstances, f.pool.MinServing = spec.DesiredInstances, spec.MinServing + f.pool.MaxSurge, f.pool.MaxRepair = spec.MaxSurge, spec.MaxRepair + f.pool.DesiredReleaseID = spec.DesiredReleaseID + return nil +} + +func (f *fakePoolStore) FreezeTrinoPool(_ context.Context, _ configstore.TrinoPoolLease, _, reason string) error { + f.frozen = reason + f.pool.Frozen, f.pool.FrozenReason = true, reason + return nil +} + +func (f *fakePoolStore) ThawTrinoPool(context.Context, configstore.TrinoPoolLease, string) error { + f.pool.Frozen, f.pool.FrozenReason = false, "" + return nil +} + +func (f *fakePoolStore) GetTrinoPool(context.Context, string) (*configstore.TrinoPool, error) { + return f.pool, nil +} + +func (f *fakePoolStore) AcquireTrinoPoolAuthority(_ context.Context, poolID, owner string) (configstore.TrinoPoolLease, error) { + f.epoch++ + f.pool.AuthorityEpoch, f.pool.AuthorityOwner = f.epoch, owner + return configstore.TrinoPoolLease{PoolID: poolID, Owner: owner, Epoch: f.epoch}, nil +} + +func (f *fakePoolStore) ListTrinoPoolInstances(context.Context, string) ([]configstore.TrinoPoolInstance, error) { + instances := make([]configstore.TrinoPoolInstance, 0, len(f.order)) + for _, id := range f.order { + instances = append(instances, *f.instances[id]) + } + return instances, nil +} + +func (f *fakePoolStore) CreateTrinoPoolInstance(_ context.Context, lease configstore.TrinoPoolLease, spec configstore.TrinoPoolInstanceSpec) error { + if lease.Epoch != f.epoch { + return configstore.ErrTrinoPoolConflict + } + if _, exists := f.instances[spec.InstanceID]; exists { + return fmt.Errorf("instance %s already exists", spec.InstanceID) + } + f.instances[spec.InstanceID] = &configstore.TrinoPoolInstance{ + InstanceID: spec.InstanceID, PoolID: spec.PoolID, ReleaseID: spec.ReleaseID, + SpecDigest: spec.SpecDigest, BlueprintSnapshot: spec.BlueprintSnapshot, + Phase: string(spec.Phase), Repair: spec.Repair, EndpointURL: spec.EndpointURL, + ValidationReceipt: "{}", RetirementReceipt: "{}", + } + f.order = append(f.order, spec.InstanceID) + return nil +} + +func (f *fakePoolStore) AdvanceTrinoPoolInstance(_ context.Context, lease configstore.TrinoPoolLease, instanceID string, from, to trinopool.Phase, updates map[string]any) error { + if f.failAdvance || lease.Epoch != f.epoch { + return configstore.ErrTrinoPoolConflict + } + if err := trinopool.ValidateTransition(from, to); err != nil { + return err + } + instance, exists := f.instances[instanceID] + if !exists || instance.Phase != string(from) { + return configstore.ErrTrinoPoolConflict + } + instance.Phase = string(to) + // The store stamps this on every transition; the operator's failure timers + // measure from it, so a fake that left it alone would make a member look + // like it had been in its new phase since whenever it entered the previous + // one. + instance.PhaseChangedAt = time.Now().UTC() + applyFakeUpdates(instance, updates) + return nil +} + +func (f *fakePoolStore) RecordTrinoPoolInstanceFields(_ context.Context, lease configstore.TrinoPoolLease, instanceID string, updates map[string]any) error { + if lease.Epoch != f.epoch { + return configstore.ErrTrinoPoolConflict + } + applyFakeUpdates(f.instances[instanceID], updates) + return nil +} + +func applyFakeUpdates(instance *configstore.TrinoPoolInstance, updates map[string]any) { + for key, value := range updates { + switch key { + case "service_name": + instance.ServiceName = value.(string) + case "service_uid": + instance.ServiceUID = value.(string) + case "config_map_name": + instance.ConfigMapName = value.(string) + case "config_map_uid": + instance.ConfigMapUID = value.(string) + case "coordinator_deployment_name": + instance.CoordinatorDeploymentName = value.(string) + case "coordinator_deployment_uid": + instance.CoordinatorDeploymentUID = value.(string) + case "worker_deployment_name": + instance.WorkerDeploymentName = value.(string) + case "worker_deployment_uid": + instance.WorkerDeploymentUID = value.(string) + case "coordinator_pod_uid": + instance.CoordinatorPodUID = value.(string) + case "coordinator_node_id": + instance.CoordinatorNodeID = value.(string) + case "coordinator_id": + instance.CoordinatorID = value.(string) + case "coordinator_boot_id": + instance.CoordinatorBootID = value.(string) + case "coordinator_container_id": + instance.CoordinatorContainerID = value.(string) + case "gateway_incarnation": + instance.GatewayIncarnation = value.(string) + case "gateway_backend_name": + instance.GatewayBackendName = value.(string) + case "gateway_state": + instance.GatewayState = value.(string) + case "gateway_generation": + instance.GatewayGeneration = value.(int64) + case "applied_catalog_revision": + instance.AppliedCatalogRevision = value.(int64) + case "repair_for": + instance.RepairFor = value.(string) + case "failure_reason": + instance.FailureReason = value.(string) + case "worker_config_map_name": + instance.WorkerConfigMapName = value.(string) + case "worker_config_map_uid": + instance.WorkerConfigMapUID = value.(string) + case "last_error": + instance.LastError = value.(string) + case "validation_receipt": + instance.ValidationReceipt = value.(string) + case "retirement_receipt": + instance.RetirementReceipt = value.(string) + } + } +} + +type fakePoolGateway struct { + members map[string]*trinogateway.Member + obligations map[string]trinogateway.Obligations + backends map[string]trinogateway.Backend + principals map[string][]string + calls []string + drainErr error + admitErr error + lostErr error + membership int64 + // lastLost is the loss claim the Gateway recorded, so a test can assert + // WHICH evidence a repair rested on. + lastLost trinogateway.LostMemberRequest + principalErr map[string]error + publications map[string]*fakePublication + admitted map[string]string + revoked map[string]bool + configured *trinogateway.ConfigurePoolRequest + + // memberJournal mirrors the Gateway's request journal for the MEMBER + // lifecycle steps, keyed by step identity. The recorded payload INCLUDES + // the expected generation, because the Gateway hashes the whole request + // body: a repeat under the same step id carrying a different generation is + // POOL_INTENT_CHANGED, not a replay. It also records the response, so a + // test can make an effect land while the caller sees a transport failure. + // + // The publication steps use `journal` + guardStep/recordStep below. They + // are kept separate because they answer different questions: this one is + // about resolving a lost response, that one about refusing a changed + // intent. + memberJournal map[string]fakeJournalEntry + // loseResponse names step ids whose effect must land while the caller sees + // a transport failure - the ambiguity every lifecycle retry has to survive. + loseResponse map[string]bool + + journal map[string]fakeStep + principalOf map[string]string + clock int64 + // deferPublish holds the next publication for a tenant at the server: the + // caller sees a lost response, and the effect commits when deliverDeferred + // runs. + deferPublish map[string]bool + deferred []func() + deferRevoke map[string]bool + // intentConflicts counts refusals of a step identity that carries content + // the journal did not record for it. Settling a request must never depend + // on provoking one: it is an anomaly, not a protocol. + intentConflicts int + // minServing is the floor the Gateway itself enforces on open and commit, + // taken from the pool configuration the operator publishes. + minServing int64 +} + +type fakeJournalEntry struct { + payload string + response trinogateway.Member +} + +func newFakePoolGateway() *fakePoolGateway { + return &fakePoolGateway{ + members: map[string]*trinogateway.Member{}, + obligations: map[string]trinogateway.Obligations{}, + backends: map[string]trinogateway.Backend{}, + } +} + +// replay answers a repeated step from the journal, exactly as the Gateway does. +// A repeat with a DIFFERENT payload is refused: that is the failure mode a +// retry hits when it rebuilds its request from freshly read state instead of +// from what it recorded when it first formed the intent. +func (f *fakePoolGateway) replay(step trinogateway.Step, payload string) (trinogateway.Member, bool, error) { + entry, recorded := f.memberJournal[step.OperationID+"/"+step.StepID] + if !recorded { + return trinogateway.Member{}, false, nil + } + if entry.payload != payload { + return trinogateway.Member{}, true, fmt.Errorf("%w: recorded %s, received %s", + trinogateway.ErrIntentChanged, entry.payload, payload) + } + return entry.response, true, nil +} + +// commit records the outcome and then, when the test asked for it, hides the +// response from the caller. +func (f *fakePoolGateway) commit(step trinogateway.Step, payload string, member trinogateway.Member) (trinogateway.Member, error) { + if f.memberJournal == nil { + f.memberJournal = map[string]fakeJournalEntry{} + } + f.memberJournal[step.OperationID+"/"+step.StepID] = fakeJournalEntry{payload: payload, response: member} + if f.loseResponse[step.StepID] { + delete(f.loseResponse, step.StepID) + return trinogateway.Member{}, errors.New("connection reset before the response was read") + } + return member, nil +} + +// fakeRequestPayload mirrors what the Gateway actually hashes under a step +// identity: the WHOLE canonical request body, minus the authority envelope it +// strips (controllerEpoch, ownerIdentity) so a successor leader can resume a +// step its predecessor committed. +// +// Hashing a stand-in - the call name and the expected generation - modelled +// generation drift and nothing else, so a retry that rebuilt any OTHER field +// (a re-stamped observation time, a re-read boot identity, a reason chosen by +// whichever check fired first) looked like a clean replay here and was refused +// by the real Gateway. +func fakeRequestPayload(request any) string { + encoded, err := json.Marshal(request) + if err != nil { + panic(fmt.Sprintf("marshal gateway request: %v", err)) + } + var body map[string]any + if err := json.Unmarshal(encoded, &body); err != nil { + panic(fmt.Sprintf("decode gateway request: %v", err)) + } + delete(body, "controllerEpoch") + delete(body, "ownerIdentity") + canonical, err := json.Marshal(body) + if err != nil { + panic(fmt.Sprintf("canonicalize gateway request: %v", err)) + } + return string(canonical) +} + +func (f *fakePoolGateway) record(call string) { f.calls = append(f.calls, call) } + +func (f *fakePoolGateway) EnsureInactiveBackend(_ context.Context, backend trinogateway.Backend) error { + f.record("backend:" + backend.Name) + if backend.Active { + return errors.New("a pooled backend must be inactive") + } + f.backends[backend.Name] = backend + return nil +} + +func (f *fakePoolGateway) PublishTenantPrincipals(_ context.Context, _, tenant string, request trinogateway.PublishPrincipalsRequest) (trinogateway.TenantAdmission, error) { + f.record("principals:" + tenant) + if err := f.principalErr[tenant]; err != nil { + return trinogateway.TenantAdmission{}, err + } + if request.Revision == "" || len(request.Principals) == 0 { + return trinogateway.TenantAdmission{}, fmt.Errorf("%w: a binding needs a revision and at least one principal", trinogateway.ErrValidation) + } + if f.deferPublish[tenant] { + // The request REACHED the Gateway and is still executing there - behind + // the pool row lock, say - while the caller's response is lost. Its + // effect commits later, in arrival order at the Gateway, and is subject + // to the journal at THAT moment rather than at call time. This is the + // case an immediate retry of identical bytes does not cover. + delete(f.deferPublish, tenant) + captured := request + f.deferred = append(f.deferred, func() { _, _ = f.applyPublish(tenant, captured) }) + return trinogateway.TenantAdmission{}, fmt.Errorf("%w: the response was lost", trinogateway.ErrUnavailable) + } + return f.applyPublish(tenant, request) +} + +// deliverDeferred commits the effects that were left executing at the Gateway, +// in the order they arrived. +func (f *fakePoolGateway) deliverDeferred() { + deferred := f.deferred + f.deferred = nil + for _, apply := range deferred { + apply() + } +} + +func (f *fakePoolGateway) applyPublish(tenant string, request trinogateway.PublishPrincipalsRequest) (trinogateway.TenantAdmission, error) { + intent := request.Revision + "|" + strings.Join(request.Principals, ",") + replayed, err := f.guardStep(request.Step, intent) + if err != nil { + return trinogateway.TenantAdmission{}, err + } + if replayed { + // PoolStore.inPool resolves the recorded step and applies NOTHING: the + // principal rows keep whatever the earlier publication left there. + return trinogateway.TenantAdmission{Tenant: tenant, State: "PENDING", PrincipalRevision: request.Revision}, nil + } + if f.principals == nil { + f.principals = map[string][]string{} + } + if f.principalOf == nil { + f.principalOf = map[string]string{} + } + // pool_tenant_principal is one flat namespace per pool: a principal already + // bound to another tenant is a conflict, never an ambiguous admission. + for _, principal := range request.Principals { + if owner, bound := f.principalOf[principal]; bound && owner != tenant { + return trinogateway.TenantAdmission{}, fmt.Errorf("%w: %s already belongs to %s", + trinogateway.ErrPrincipalConflict, principal, owner) + } + } + for _, principal := range f.principals[tenant] { + delete(f.principalOf, principal) + } + for _, principal := range request.Principals { + f.principalOf[principal] = tenant + } + f.principals[tenant] = request.Principals + f.recordStep(request.Step, intent) + return trinogateway.TenantAdmission{Tenant: tenant, State: "PENDING", PrincipalRevision: request.Revision}, nil +} + +func (f *fakePoolGateway) ConfigurePool(_ context.Context, _ string, request trinogateway.ConfigurePoolRequest) (trinogateway.PoolState, error) { + f.record("configure") + f.configured = &request + f.minServing = int64(request.MinServing) + return trinogateway.PoolState{}, nil +} + +func (f *fakePoolGateway) RegisterMember(_ context.Context, poolID string, request trinogateway.RegisterMemberRequest) (trinogateway.Member, error) { + f.record("register:" + request.InstanceID) + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + // The Gateway probes the coordinator itself at registration and binds the + // member to the identity it observed, so the response - not the request - + // is where those values come from. + member := &trinogateway.Member{ + PoolID: poolID, InstanceID: request.InstanceID, BackendName: request.BackendName, + Incarnation: "incarnation-" + request.InstanceID, Phase: "PREPARING", Generation: 1, + NodeID: "node-1", CoordinatorID: "abcde", + // Bound at registration, and a later receipt or loss claim has to + // present the identical pair. + PodUID: request.PodUID, BootID: request.BootID, + } + f.members[request.InstanceID] = member + return f.commit(request.Step, payload, *member) +} + +func (f *fakePoolGateway) AdmitMember(_ context.Context, _, instanceID string, request trinogateway.AdmitMemberRequest) (trinogateway.Member, error) { + f.record("admit:" + instanceID) + if f.admitErr != nil { + return trinogateway.Member{}, f.admitErr + } + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + member := f.members[instanceID] + if err := requireGeneration(member, request.ExpectedGeneration); err != nil { + return trinogateway.Member{}, err + } + if member == nil { + return trinogateway.Member{}, fmt.Errorf("%w: %s", trinogateway.ErrNotFound, instanceID) + } + // PoolStore.admitMember: a member joining while ANY publication is open must + // acknowledge that publication's target revision. A candidate registers + // under its RELEASE id, so it never can - which is what makes an open + // barrier block the pool's own compute lifecycle. + if barrier := f.oldestOpenPublication(); barrier != nil { + if request.Receipt.ConfigRevision != barrier.TargetRevision { + return trinogateway.Member{}, fmt.Errorf( + "%w: a member joining during a publication must acknowledge its target revision", + trinogateway.ErrPublicationBarrier) + } + barrier.received[instanceID] = fakeReceipt{bootID: member.BootID, fingerprint: request.Receipt.CertificateHash} + } + member.Phase, member.Generation, member.Eligible = "ACTIVE", member.Generation+1, true + // Admission changes the serving set, so the membership generation moves and + // every open barrier's commit CAS now fails. + f.membership++ + return f.commit(request.Step, payload, *member) +} + +func (f *fakePoolGateway) GetMember(_ context.Context, _, instanceID string) (trinogateway.Member, error) { + member, exists := f.members[instanceID] + if !exists { + // The real client maps the Gateway's POOL_NOT_FOUND to this sentinel, + // and a read-back that resolves a lost response has to tell "nothing was + // ever recorded" from "the Gateway could not be reached". + return trinogateway.Member{}, fmt.Errorf("%w: %s", trinogateway.ErrNotFound, instanceID) + } + return *member, nil +} + +func (f *fakePoolGateway) GetObligations(_ context.Context, _, instanceID string) (trinogateway.Obligations, error) { + return f.obligations[instanceID], nil +} + +// requirePhase mirrors the Gateway's own phase preconditions. Without them a +// fake accepts transitions the real PoolStore refuses with POOL_PHASE, and the +// tests prove the operator can drive a protocol nobody implements. +func (f *fakePoolGateway) requirePhase(instanceID, call string, allowed ...string) (*trinogateway.Member, error) { + member, known := f.members[instanceID] + if !known { + return nil, fmt.Errorf("%w: %s", trinogateway.ErrNotFound, instanceID) + } + if !slices.Contains(allowed, member.Phase) { + return nil, fmt.Errorf("%w: a %s member cannot %s", trinogateway.ErrPhase, member.Phase, call) + } + return member, nil +} + +// requireGeneration mirrors the member CAS. A step carrying a stale generation +// is refused, which is what makes a read-back before each step necessary rather +// than optional. +func requireGeneration(member *trinogateway.Member, expected int64) error { + if member.Generation != expected { + return fmt.Errorf("%w: member is at generation %d, step carries %d", + trinogateway.ErrStaleGeneration, member.Generation, expected) + } + return nil +} + +func (f *fakePoolGateway) DrainMember(_ context.Context, _, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) { + f.record("drain:" + instanceID) + if f.drainErr != nil { + return trinogateway.Member{}, f.drainErr + } + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + // ACTIVE is the planned drain. SUSPECT is the extension agreed with the + // Gateway for a member that is excluded but not provably dead: it is the + // only way such a member can ever leave, since a loss claim needs evidence + // a crash-looping pod never provides. + member, err := f.requirePhase(instanceID, "be drained", "ACTIVE", "SUSPECT") + if err != nil { + return trinogateway.Member{}, err + } + if err := requireGeneration(member, request.ExpectedGeneration); err != nil { + return trinogateway.Member{}, err + } + member.Phase, member.Generation = "DRAINING", member.Generation+1 + f.membership++ + return f.commit(request.Step, payload, *member) +} + +func (f *fakePoolGateway) SealMember(_ context.Context, _, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) { + f.record("seal:" + instanceID) + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + member, err := f.requirePhase(instanceID, "be sealed", "DRAINING") + if err != nil { + return trinogateway.Member{}, err + } + if err := requireGeneration(member, request.ExpectedGeneration); err != nil { + return trinogateway.Member{}, err + } + member.Phase, member.Generation = "SEALED", member.Generation+1 + // Obligations are reported with the member's CURRENT generation, so a step + // that rebuilds its request from a fresh obligations read carries a + // different generation than the one the journal recorded. + obligations := f.obligations[instanceID] + obligations.Generation = member.Generation + f.obligations[instanceID] = obligations + return f.commit(request.Step, payload, *member) +} + +func (f *fakePoolGateway) SuspectMember(_ context.Context, _, instanceID string, request trinogateway.SuspectMemberRequest) (trinogateway.Member, error) { + f.record("suspect:" + instanceID) + if request.Reason == "" { + return trinogateway.Member{}, errors.New("a suspicion must carry a reason") + } + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + member, err := f.requirePhase(instanceID, "become suspect", "PREPARING", "ACTIVE", "DRAINING", "SEALED") + if err != nil { + return trinogateway.Member{}, err + } + if err := requireGeneration(member, request.ExpectedGeneration); err != nil { + return trinogateway.Member{}, err + } + wasActive := member.Phase == "ACTIVE" + member.Phase, member.Generation = "SUSPECT", member.Generation+1 + if wasActive { + f.membership++ + } + return f.commit(request.Step, payload, *member) +} + +func (f *fakePoolGateway) LostMember(_ context.Context, _, instanceID string, request trinogateway.LostMemberRequest) (trinogateway.Member, error) { + f.record("lost:" + instanceID) + if f.lostErr != nil { + return trinogateway.Member{}, f.lostErr + } + if request.Evidence == "" || request.Termination.Source == "" { + return trinogateway.Member{}, errors.New("a loss claim needs termination evidence") + } + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + member, err := f.requirePhase(instanceID, "be declared lost", "SUSPECT") + if err != nil { + return trinogateway.Member{}, err + } + if err := requireGeneration(member, request.ExpectedGeneration); err != nil { + return trinogateway.Member{}, err + } + // The evidence must identify the exact incarnation the Gateway recorded. + if request.Termination.PodUID != member.PodUID || request.Termination.BootID != member.BootID || + request.Termination.NodeID != member.NodeID || request.Termination.CoordinatorID != member.CoordinatorID { + return trinogateway.Member{}, fmt.Errorf("%w: termination evidence does not identify this incarnation", + trinogateway.ErrEvidenceRequired) + } + member.Phase, member.Generation, member.RetirementKind = "LOST", member.Generation+1, "FAILED" + f.lastLost = request + return f.commit(request.Step, payload, *member) +} + +func (f *fakePoolGateway) RetireMember(_ context.Context, _, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) { + f.record("retire:" + instanceID) + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + // SEALED is a completed drain, LOST a proven failure, and PREPARING a + // candidate that never admitted work. Nothing else may claim retirement. + member, err := f.requirePhase(instanceID, "be retired", "SEALED", "LOST", "PREPARING") + if err != nil { + return trinogateway.Member{}, err + } + if err := requireGeneration(member, request.ExpectedGeneration); err != nil { + return trinogateway.Member{}, err + } + kind := "DRAINED" + if member.Phase == "LOST" { + kind = "FAILED" + } + member.Phase, member.Generation, member.RetirementKind = "RETIRING", member.Generation+1, kind + return f.commit(request.Step, payload, *member) +} + +func (f *fakePoolGateway) MemberRetired(_ context.Context, _, instanceID string, request trinogateway.MemberStepRequest) (trinogateway.Member, error) { + f.record("retired:" + instanceID) + if !request.ResourcesAbsent { + return trinogateway.Member{}, errors.New("retirement reported without asserting absence") + } + payload := fakeRequestPayload(request) + if replayed, done, err := f.replay(request.Step, payload); done { + return replayed, err + } + member, err := f.requirePhase(instanceID, "complete retirement", "RETIRING") + if err != nil { + return trinogateway.Member{}, err + } + if err := requireGeneration(member, request.ExpectedGeneration); err != nil { + return trinogateway.Member{}, err + } + member.Phase, member.Generation = "RETIRED", member.Generation+1 + return f.commit(request.Step, payload, *member) +} + +type fakePoolKube struct { + applied map[string]trinoPoolInventory + deleted map[string]bool + observed trinoPoolObservation + absent bool + // absentAfterDelete makes deletion actually remove the objects, which is + // what lets a test drive a teardown to its end rather than asserting only + // the step that starts it. + absentAfterDelete bool + epochsSeen []int64 +} + +func newFakePoolKube() *fakePoolKube { + return &fakePoolKube{ + applied: map[string]trinoPoolInventory{}, + deleted: map[string]bool{}, + observed: trinoPoolObservation{ + CoordinatorReady: true, ReadyWorkers: 4, DesiredWorkers: 4, + CoordinatorPodUID: "pod-uid-1", PodsPresent: 5, + // A real Observe always reports the coordinator pod it read the + // readiness from, including which container instance is running - + // that is what registration records so a later termination record + // can be correlated with the admitted process. + CoordinatorPods: []trinoPoolCoordinatorPod{ + {UID: "pod-uid-1", RunningContainerID: "containerd://admitted"}, + }, + }, + } +} + +func (f *fakePoolKube) forEpoch(epoch int64) trinoPoolKube { + f.epochsSeen = append(f.epochsSeen, epoch) + return f +} + +func (f *fakePoolKube) Apply(_ context.Context, objects trinopool.Objects) (trinoPoolInventory, error) { + inventory := trinoPoolInventory{ + Namespace: objects.Service.Namespace, + ConfigMapName: objects.ConfigMap.Name, + ConfigMapUID: "cm-uid", + ServiceName: objects.Service.Name, + ServiceUID: "svc-uid", + CoordinatorDeploymentName: objects.CoordinatorDeployment.Name, + CoordinatorDeploymentUID: "coord-uid", + WorkerDeploymentName: objects.WorkerDeployment.Name, + WorkerDeploymentUID: "worker-uid", + } + f.applied[objects.Service.Name] = inventory + return inventory, nil +} + +func (f *fakePoolKube) Observe(context.Context, trinoPoolInventory) (trinoPoolObservation, error) { + return f.observed, nil +} + +func (f *fakePoolKube) Delete(_ context.Context, inventory trinoPoolInventory) error { + f.deleted[inventory.ServiceName] = true + return nil +} + +func (f *fakePoolKube) ResourcesAbsent(_ context.Context, inventory trinoPoolInventory) (bool, error) { + if f.absentAfterDelete && f.deleted[inventory.ServiceName] { + return true, nil + } + return f.absent, nil +} + +// --------------------------------------------------------------------------- +// The loop. +// --------------------------------------------------------------------------- + +type operatorHarness struct { + operator *trinoPoolOperator + store *fakePoolStore + gateway *fakePoolGateway + kube *fakePoolKube + publications *fakePublicationStore +} + +func newOperatorHarness(t *testing.T) *operatorHarness { + t.Helper() + blueprint, err := trinopool.ParseBlueprint(testBlueprintJSON(t)) + if err != nil { + t.Fatalf("parse blueprint: %v", err) + } + config := trinoPoolConfig{ + PoolID: "registered:cell-001", PublicID: "cell-001", + RoutingGroup: "cell-001", Namespace: blueprint.Namespace, + Blueprint: blueprint, + Pool: trinoRegisteredPool{ + DesiredInstances: 3, MinServing: 3, MaxSurge: 1, MaxRepair: 1, + CoordinatorServicePort: 8443, NodeEnvironment: "mw_dev_pool_001", + }, + Spec: configstore.TrinoPoolSpec{ + PoolID: "registered:cell-001", PublicID: "cell-001", + APIMode: configstore.TrinoPoolAPIModeShared, DesiredInstances: 3, + MinServing: 3, MaxSurge: 1, MaxRepair: 1, + DesiredReleaseID: blueprint.ReleaseID, DesiredBlueprintDigest: blueprint.Digest(), + }, + } + store := newFakePoolStore(config.Spec) + gateway := newFakePoolGateway() + kube := newFakePoolKube() + + sequence := 0 + publications := newFakePublicationStore() + return &operatorHarness{ + store: store, gateway: gateway, kube: kube, publications: publications, + operator: &trinoPoolOperator{ + config: config, store: store, gateway: gateway, + kube: kube.forEpoch, + owner: "cp-test", + operatorEnabled: true, + validate: func(_ context.Context, _ string, _ trinoPoolObservation, _ trinoPoolExpectation) (trinoPoolValidation, error) { + return trinoPoolValidation{ + NodeID: "node-1", ProcessID: "process-1", CoordinatorID: "abcde", + AppliedRevision: 42, AuthRevision: "auth", ReadyWorkers: 4, + Checks: []string{trinoPoolCheckImage}, CertificateHash: "hash", + }, nil + }, + identity: func(context.Context, string) (string, error) { return "process-1", nil }, + publications: publications, + // Every member is serving the projection the control plane is + // publishing. Tests that need the opposite override this. + acknowledgement: func(_ context.Context, _ string, _ trinoPoolProjectionRevisions, _ int64) (trinoPoolAcknowledgement, error) { + return trinoPoolAcknowledgement{ProcessID: "process-1", AppliedRevision: 42, ProjectionCurrent: true}, nil + }, + projection: func() trinoPoolProjectionRevisions { + return trinoPoolProjectionRevisions{Policy: "policy-1", Password: "password-1", Group: "group-1"} + }, + newInstanceID: func() string { + sequence++ + return fmt.Sprintf("%08x", sequence) + }, + }, + } +} + +func (h *operatorHarness) tick(t *testing.T, times int) { + t.Helper() + for index := 0; index < times; index++ { + if err := h.operator.reconcileOnce(context.Background()); err != nil { + t.Fatalf("tick %d: %v", index, err) + } + } +} + +// placeInstance puts an instance into a phase on BOTH sides. The Gateway +// enforces its own phase preconditions, so a test that moved only the local row +// would be driving a protocol the real Gateway refuses. +func (h *operatorHarness) placeInstance(t *testing.T, instanceID string, local trinopool.Phase, gatewayPhase string) *configstore.TrinoPoolInstance { + t.Helper() + instance, known := h.store.instances[instanceID] + if !known { + t.Fatalf("instance %s does not exist", instanceID) + } + instance.Phase = string(local) + member, known := h.gateway.members[instanceID] + if !known { + t.Fatalf("instance %s has no gateway member", instanceID) + } + member.Phase = gatewayPhase + instance.GatewayGeneration = member.Generation + return instance +} + +// tickTolerant runs ticks that are EXPECTED to fail, which is what a refusing +// Gateway produces. The reconcile loop logs and carries on; these tests assert +// what was recorded while it did. +func (h *operatorHarness) tickTolerant(times int) { + for index := 0; index < times; index++ { + _ = h.operator.reconcileOnce(context.Background()) + } +} + +func (h *operatorHarness) phases() map[string]string { + phases := map[string]string{} + for id, instance := range h.store.instances { + phases[id] = instance.Phase + } + return phases +} + +// With the operator disabled the desired state is still recorded, and nothing +// in Kubernetes or the Gateway is touched. That is what shipping this feature +// disabled has to mean: the wiring exists and stays inert. +func TestOperatorDisabledTouchesNothingExternal(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.operatorEnabled = false + + harness.tick(t, 3) + + if harness.store.pool.DesiredInstances != 3 { + t.Fatal("the desired spec was not recorded") + } + if len(harness.gateway.calls) != 0 { + t.Fatalf("the disabled operator called the gateway: %v", harness.gateway.calls) + } + if len(harness.kube.applied) != 0 { + t.Fatalf("the disabled operator created %d instances", len(harness.kube.applied)) + } + if harness.operator.lease.Epoch != 0 { + t.Fatal("the disabled operator claimed authority") + } +} + +// A frozen pool holds its last-good state: nothing is created, nothing is +// deleted, and the freeze reason reaches the durable record for the operator. +func TestFrozenPoolMakesNoExternalChanges(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Frozen = true + harness.operator.config.FrozenReason = "blueprint is unreadable" + + harness.tick(t, 2) + + if harness.store.frozen != "blueprint is unreadable" { + t.Fatalf("freeze reason = %q", harness.store.frozen) + } + if len(harness.kube.applied) != 0 || len(harness.gateway.calls) != 0 { + t.Fatal("a frozen pool produced external effects") + } +} + +// The full happy path: three instances reach SERVING, each through create, +// register, validate, admit. +func TestOperatorBringsThePoolToTheDesiredCount(t *testing.T) { + harness := newOperatorHarness(t) + // Each instance needs several ticks; the loop deliberately advances one + // instance per tick. + harness.tick(t, 20) + + serving := 0 + for _, phase := range harness.phases() { + if phase == string(trinopool.PhaseServing) { + serving++ + } + } + if serving != 3 { + t.Fatalf("phases = %v, want three serving", harness.phases()) + } + // Registration must have created the backend record first and left it + // inactive; otherwise the member would be routable before it is certified. + for name, backend := range harness.gateway.backends { + if backend.Active { + t.Fatalf("backend %s was registered active", name) + } + } + if len(harness.gateway.backends) != 3 { + t.Fatalf("registered %d backends", len(harness.gateway.backends)) + } +} + +// A pool at its desired count and release does nothing further. A reconcile +// loop that keeps acting on a converged pool is how fleets get churned. +func TestOperatorIsQuietWhenConverged(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + before := len(harness.gateway.calls) + + harness.tick(t, 5) + after := len(harness.gateway.calls) + + // Only the idempotent pool configuration call repeats. + for _, call := range harness.gateway.calls[before:after] { + if call != "configure" { + t.Fatalf("a converged pool issued %q", call) + } + } + if len(harness.kube.applied) != 3 { + t.Fatalf("a converged pool created %d instances", len(harness.kube.applied)) + } +} + +// A new release surges ONE replacement, and only drains an old instance once +// the replacement is actually serving. +func TestOperatorSurgesThenDrainsForANewRelease(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + + harness.store.pool.DesiredReleaseID = "next-release" + harness.operator.config.Spec.DesiredReleaseID = "next-release" + harness.operator.config.Blueprint.ReleaseID = "next-release" + + // One tick creates the surge instance; it then needs ticks to reach + // SERVING before any drain may start. + harness.tick(t, 1) + if drained := countCalls(harness.gateway.calls, "drain:"); drained != 0 { + t.Fatal("a drain started before the replacement was serving") + } + if len(harness.store.instances) != 4 { + t.Fatalf("surge created %d instances", len(harness.store.instances)) + } + + harness.tick(t, 10) + if drained := countCalls(harness.gateway.calls, "drain:"); drained != 1 { + t.Fatalf("expected exactly one drain, got %d", drained) + } +} + +// The Gateway's serving-floor refusal is authoritative. The operator records it +// and retries later; it never forces the drain. +func TestServingFloorRefusalIsNotOverridden(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + harness.gateway.drainErr = trinogateway.ErrServingFloor + + harness.store.pool.DesiredReleaseID = "next-release" + harness.operator.config.Spec.DesiredReleaseID = "next-release" + harness.operator.config.Blueprint.ReleaseID = "next-release" + // The refusal surfaces as an error from the tick that attempts the drain, + // so these ticks are allowed to fail. + for index := 0; index < 11; index++ { + _ = harness.operator.reconcileOnce(context.Background()) + } + + err := harness.operator.reconcileOnce(context.Background()) + if err == nil || !errors.Is(err, trinogateway.ErrServingFloor) { + t.Fatalf("error = %v, want the serving-floor refusal to surface", err) + } + for _, instance := range harness.store.instances { + if instance.Phase == string(trinopool.PhaseDraining) { + t.Fatal("an instance was drained despite the refusal") + } + } +} + +// Nothing is deleted before the Gateway has irreversibly claimed retirement, +// and retirement completes only once the resources are verifiably absent. +func TestRetirementRequiresAClaimThenVerifiedAbsence(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseSealed, "SEALED") + + // Sealed -> the operator claims retirement. Still nothing deleted. + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseRetiring) { + t.Fatalf("phase = %s, want RETIRING", instance.Phase) + } + if len(harness.kube.deleted) != 0 { + t.Fatal("resources were deleted before the retirement claim") + } + if instance.RetirementReceipt == "{}" || instance.RetirementReceipt == "" { + t.Fatal("the retirement claim was not recorded") + } + + // Deletion starts, but the pods are still terminating: the instance stays + // RETIRING and keeps its slot. + harness.kube.absent = false + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseRetiring) { + t.Fatalf("phase = %s, want the instance to hold RETIRING while pods remain", instance.Phase) + } + if countCalls(harness.gateway.calls, "retired:") != 0 { + t.Fatal("retirement was reported before the resources were gone") + } + + harness.kube.absent = true + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseRetired) { + t.Fatalf("phase = %s, want RETIRED", instance.Phase) + } +} + +// A member is not sealed while anything is still pinned to it. There is no +// drain deadline: sealing on a timer would be a decision to lose that work. +func TestSealWaitsForObligations(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseDraining, "DRAINING") + generation := harness.gateway.members[instanceID].Generation + harness.gateway.obligations[instanceID] = trinogateway.Obligations{ + Generation: generation, OpenTransactions: 1, Drained: false, + } + + harness.tick(t, 3) + if instance.Phase != string(trinopool.PhaseDraining) { + t.Fatalf("phase = %s, want the instance to stay DRAINING", instance.Phase) + } + if countCalls(harness.gateway.calls, "seal:") != 0 { + t.Fatal("a member with an open transaction was sealed") + } + + harness.gateway.obligations[instanceID] = trinogateway.Obligations{Generation: generation, Drained: true} + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseSealed) { + t.Fatalf("phase = %s, want SEALED once drained", instance.Phase) + } +} + +// A lost response must not change the request the retry sends. +// +// The Gateway journals each step under its identity and hashes the whole +// request, expected generation included. So a step whose effect LANDED while +// its response was lost can only be resolved by repeating the identical +// request: rebuilding it from freshly read state sends the generation the +// committed effect produced, the journal reports a changed intent, and that +// member can never finish the transition - not on the next tick, not after a +// leader change, never. +func TestALostResponseDoesNotChangeTheRetriedRequest(t *testing.T) { + t.Run("admit", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 2) + instanceID := harness.store.order[0] + harness.gateway.loseResponse = map[string]bool{"admit": true} + + harness.tickTolerant(20) + + instance := harness.store.instances[instanceID] + if instance.Phase != string(trinopool.PhaseServing) { + t.Fatalf("phase = %s, want the admission to be resolved from the journal", instance.Phase) + } + if member := harness.gateway.members[instanceID]; member.Phase != "ACTIVE" { + t.Fatalf("gateway phase = %s, want the admitted member to stay ACTIVE", member.Phase) + } + }) + + t.Run("seal", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseDraining, "DRAINING") + harness.gateway.obligations[instanceID] = trinogateway.Obligations{ + Generation: harness.gateway.members[instanceID].Generation, Drained: true, + } + harness.gateway.loseResponse = map[string]bool{"seal": true} + + harness.tickTolerant(5) + + if instance.Phase != string(trinopool.PhaseSealed) && instance.Phase != string(trinopool.PhaseRetiring) { + t.Fatalf("phase = %s, want the seal to be resolved from the journal", instance.Phase) + } + }) +} + +// Losing the authority CAS means this leader has been superseded. It must stop +// writing and re-acquire, not retry the same epoch harder. +func TestLostAuthorityIsDroppedAndReacquired(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 1) + first := harness.operator.lease.Epoch + + harness.store.failAdvance = true + if err := harness.operator.reconcileOnce(context.Background()); err == nil { + t.Fatal("a fenced write failure was not surfaced") + } + if harness.operator.lease.Epoch != 0 { + t.Fatal("the operator kept an epoch the store refused") + } + + harness.store.failAdvance = false + harness.tick(t, 1) + if harness.operator.lease.Epoch <= first { + t.Fatalf("epoch %d did not advance past %d", harness.operator.lease.Epoch, first) + } +} + +// Every Kubernetes effect is made under the CURRENT authority epoch, so a +// superseded leader's writes are refused at the object level too. +func TestKubernetesEffectsCarryTheCurrentEpoch(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 5) + if len(harness.kube.epochsSeen) == 0 { + t.Fatal("no kubernetes effect was attempted") + } + for _, epoch := range harness.kube.epochsSeen { + if epoch != harness.operator.lease.Epoch { + t.Fatalf("an effect used epoch %d, current is %d", epoch, harness.operator.lease.Epoch) + } + } +} + +// The tenant-admission gate stays closed while the identity question is open. +// Turning it on would claim an atomic gate duckgres cannot currently back. +func TestTenantAdmissionGateStaysClosed(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 1) + if harness.gateway.configured == nil { + t.Fatal("the pool was never configured") + } + if harness.gateway.configured.TenantAdmissionEnabled { + t.Fatal("the operator enabled tenant admission") + } + if harness.gateway.configured.DesiredMembers != 3 || harness.gateway.configured.MaxRepair != 1 { + t.Fatalf("configure request = %+v", harness.gateway.configured) + } +} + +func countCalls(calls []string, prefix string) int { + total := 0 + for _, call := range calls { + if len(call) >= len(prefix) && call[:len(prefix)] == prefix { + total++ + } + } + return total +} + +// --------------------------------------------------------------------------- +// Tenant binding, failure branch and authority lifecycle. +// --------------------------------------------------------------------------- + +type fakeTenantStore struct{ orgs []configstore.TrinoEnabledOrg } + +func (f *fakeTenantStore) ListTrinoEnabledOrgs() ([]configstore.TrinoEnabledOrg, error) { + return f.orgs, nil +} + +func poolOrg(users ...string) configstore.TrinoEnabledOrg { + org := configstore.TrinoEnabledOrg{ + OrgID: "org-a", DatabaseName: "acme", + CellID: "registered:cell-001", RootPasswordHash: "hash", + } + for _, username := range users { + org.Users = append(org.Users, configstore.TrinoOrgUser{Username: username, PasswordHash: "hash"}) + } + return org +} + +// With the gate on, the binding must reach the Gateway BEFORE any lifecycle +// step: the restriction refuses work whose principal it cannot place, so a pool +// that admitted members before publishing would deny its own tenants. +func TestTenantBindingIsPublishedWhenTheGateIsOn(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + + harness.tick(t, 1) + + if harness.gateway.configured == nil || !harness.gateway.configured.TenantAdmissionEnabled { + t.Fatal("the gate knob was not passed to the Gateway") + } + published := harness.gateway.principals["org-a"] + if len(published) != 2 { + t.Fatalf("published principals = %v, want the root login and the user", published) + } + // The bare database name is the root login and carries no separator; a gate + // that inferred the tenant from a dotted prefix would refuse it. + var sawBare bool + for _, principal := range published { + if principal == "acme" { + sawBare = true + } + } + if !sawBare { + t.Fatalf("published principals = %v, want the bare root login included", published) + } +} + +// An unchanged tenant is not republished; a changed login set is, because the +// Gateway replaces the set whole and a removed login must stop being admitted. +func TestTenantBindingIsRepublishedOnlyWhenItChanges(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + + harness.tick(t, 3) + if count := countCalls(harness.gateway.calls, "principals:"); count != 1 { + t.Fatalf("published %d times for an unchanged tenant", count) + } + + tenants.orgs = []configstore.TrinoEnabledOrg{poolOrg("analyst", "dagster")} + harness.tick(t, 1) + if count := countCalls(harness.gateway.calls, "principals:"); count != 2 { + t.Fatalf("a changed login set published %d times", count) + } +} + +// With the gate off nothing is published: the pool is not making that promise. +func TestNoTenantBindingWhenTheGateIsOff(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + + harness.tick(t, 2) + if countCalls(harness.gateway.calls, "principals:") != 0 { + t.Fatal("a pool with the gate off published a binding") + } +} + +// A serving member whose coordinator is gone did NOT drain. It is excluded from +// new work first, and only declared lost once the resources are verifiably +// absent - a failing probe is not evidence of death. +func TestUnhealthyMemberIsSuspectedThenLostOnlyWithEvidence(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + + instanceID := harness.store.order[0] + instance := harness.store.instances[instanceID] + instance.PhaseChangedAt = nowUTC().Add(-time.Hour) + harness.kube.observed.CoordinatorReady = false + + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want SUSPECT", instance.Phase) + } + if countCalls(harness.gateway.calls, "lost:") != 0 { + t.Fatal("a member was declared lost on a failing probe alone") + } + + // Pods still present: nothing is declared and nothing is deleted. + harness.kube.absent = false + harness.kube.observed.PodsPresent = 3 + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want the member to stay SUSPECT while its pods exist", instance.Phase) + } + + // Verified absence is the evidence. + harness.kube.absent = true + harness.kube.observed.PodsPresent = 0 + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseLost) { + t.Fatalf("phase = %s, want LOST once the resources are gone", instance.Phase) + } + if countCalls(harness.gateway.calls, "lost:") != 1 { + t.Fatal("the loss was not recorded with the Gateway") + } + // Reported as failed, never as a clean drain. + if countCalls(harness.gateway.calls, "seal:") != 0 { + t.Fatal("a lost member was sealed as if it had drained") + } +} + +// A suspected member that starts looking healthy again is NOT returned to +// service locally. +// +// Suspicion is the Gateway's state as much as this row's: it excluded the +// member and only a fresh certified admission un-excludes it. Flipping the +// local row back to SERVING would leave duckgres believing a member serves +// while the Gateway routes nothing to it - the precise divergence the phase +// machine exists to prevent. The member leaves through the planned drain +// instead, and its replacement is certified from scratch. +func TestRecoveredSuspectIsNotReturnedToServiceLocally(t *testing.T) { + harness := newOperatorHarness(t) + harness.tick(t, 20) + + instanceID := harness.store.order[0] + instance := harness.placeInstance(t, instanceID, trinopool.PhaseSuspect, "SUSPECT") + + // Healthy again, as far as Kubernetes is concerned. + harness.tick(t, 1) + if instance.Phase != string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want the member to stay SUSPECT rather than be re-admitted locally", instance.Phase) + } + if harness.gateway.members[instanceID].Phase != "SUSPECT" { + t.Fatal("the gateway member changed phase without an admission") + } + + // The phase machine itself refuses the transition, so no future path can + // reintroduce it by accident. + if err := trinopool.ValidateTransition(trinopool.PhaseSuspect, trinopool.PhaseServing); err == nil { + t.Fatal("SUSPECT -> SERVING is permitted; a local recovery would diverge from the Gateway") + } +} + +// Desired-state publication is lifecycle-affecting, so it is fenced too. A +// read-only operator has no authority and must not write it at all. +func TestReadOnlyOperatorDoesNotPublishDesiredState(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.operatorEnabled = false + harness.store.pool.DesiredReleaseID = "someone-elses-release" + + harness.tick(t, 2) + + if harness.store.pool.DesiredReleaseID != "someone-elses-release" { + t.Fatal("a read-only operator overwrote authority-owned desired state") + } + if harness.store.epoch != 0 { + t.Fatal("a read-only operator claimed authority") + } +} + +// --------------------------------------------------------------------------- +// Durable operation recording. +// --------------------------------------------------------------------------- + +type fakeOperationStore struct { + operations map[string]configstore.TrinoPoolOperation + steps map[string]configstore.TrinoPoolOperationStep + epoch func() int64 +} + +func newFakeOperationStore(epoch func() int64) *fakeOperationStore { + return &fakeOperationStore{ + operations: map[string]configstore.TrinoPoolOperation{}, + steps: map[string]configstore.TrinoPoolOperationStep{}, + epoch: epoch, + } +} + +func (f *fakeOperationStore) BeginTrinoPoolOperation(_ context.Context, lease configstore.TrinoPoolLease, spec configstore.TrinoPoolOperationSpec) (configstore.TrinoPoolOperation, error) { + if lease.Epoch != f.epoch() { + return configstore.TrinoPoolOperation{}, configstore.ErrTrinoPoolConflict + } + if existing, ok := f.operations[spec.OperationID]; ok { + if existing.IntentHash != spec.IntentHash { + return configstore.TrinoPoolOperation{}, configstore.ErrTrinoPoolIntentChanged + } + existing.Replayed = true + return existing, nil + } + created := configstore.TrinoPoolOperation{OperationID: spec.OperationID, IntentHash: spec.IntentHash} + f.operations[spec.OperationID] = created + return created, nil +} + +func (f *fakeOperationStore) RecordTrinoPoolOperationStep(_ context.Context, lease configstore.TrinoPoolLease, operationID, stepID, payloadHash, outcome, result string) (configstore.TrinoPoolOperationStep, error) { + if lease.Epoch != f.epoch() { + return configstore.TrinoPoolOperationStep{}, configstore.ErrTrinoPoolConflict + } + key := operationID + "/" + stepID + if existing, ok := f.steps[key]; ok { + if existing.PayloadHash != payloadHash { + return configstore.TrinoPoolOperationStep{}, configstore.ErrTrinoPoolIntentChanged + } + existing.Replayed = true + // A later call with a real outcome replaces the provisional UNKNOWN. + if outcome != "" && outcome != "UNKNOWN" { + existing.Outcome, existing.Result = outcome, result + f.steps[key] = existing + } + return existing, nil + } + created := configstore.TrinoPoolOperationStep{ + OperationID: operationID, StepID: stepID, PayloadHash: payloadHash, + Outcome: outcome, Result: result, + } + f.steps[key] = created + return created, nil +} + +func (f *fakeOperationStore) FinishTrinoPoolOperation(_ context.Context, _ configstore.TrinoPoolLease, operationID, phase, lastError string) error { + operation, known := f.operations[operationID] + if !known { + return configstore.ErrTrinoPoolConflict + } + now := time.Now().UTC() + operation.Phase, operation.LastError, operation.TerminalAt = phase, lastError, &now + f.operations[operationID] = operation + return nil +} + +func (f *fakeOperationStore) UpdateTrinoPoolOperation(_ context.Context, lease configstore.TrinoPoolLease, operationID string, updates map[string]any) error { + if lease.Epoch != f.epoch() { + return configstore.ErrTrinoPoolConflict + } + operation, known := f.operations[operationID] + if !known || operation.TerminalAt != nil { + return configstore.ErrTrinoPoolConflict + } + if attempts, ok := updates["attempts"].(int64); ok { + operation.Attempts = attempts + } + if next, ok := updates["next_attempt_at"].(time.Time); ok { + operation.NextAttemptAt = &next + } + if lastError, ok := updates["last_error"].(string); ok { + operation.LastError = lastError + } + f.operations[operationID] = operation + return nil +} + +// An admission whose response is lost is UNKNOWN, not failed: the member may +// already be ACTIVE. The intent is recorded before the call, so the next +// attempt resolves it by read-back instead of deciding from nothing. +func TestAdmissionRecordsItsIntentAndOutcome(t *testing.T) { + harness := newOperatorHarness(t) + operations := newFakeOperationStore(func() int64 { return harness.store.epoch }) + harness.operator.operations = operations + + harness.tick(t, 20) + + var admitStep configstore.TrinoPoolOperationStep + for key, step := range operations.steps { + if step.StepID == "admit" { + admitStep = step + _ = key + break + } + } + if admitStep.StepID == "" { + t.Fatalf("no admit step was recorded: %v", operations.steps) + } + if admitStep.Outcome != "OK" { + t.Fatalf("admit outcome = %q, want OK once the Gateway answered", admitStep.Outcome) + } + if admitStep.PayloadHash == "" { + t.Fatal("the admit step recorded no payload identity") + } + if len(operations.operations) == 0 { + t.Fatal("no durable operation was recorded") + } +} + +// A step already recorded OK is not performed again: a second admission call +// would be a duplicate effect this controller can avoid entirely. +func TestCompletedStepIsNotRepeated(t *testing.T) { + harness := newOperatorHarness(t) + operations := newFakeOperationStore(func() int64 { return harness.store.epoch }) + harness.operator.operations = operations + harness.tick(t, 20) + + before := countCalls(harness.gateway.calls, "admit:") + instanceID := harness.store.order[0] + instance := harness.store.instances[instanceID] + // Force the instance back to the admitting step with the record intact. + instance.Phase = string(trinopool.PhaseValidating) + + harness.tick(t, 1) + if countCalls(harness.gateway.calls, "admit:") != before { + t.Fatal("a step already recorded OK was performed again") + } + if instance.Phase != string(trinopool.PhaseAdmitted) { + t.Fatalf("phase = %s, want the recorded outcome to advance the instance", instance.Phase) + } +} + +// A candidate that can never be admitted must not be abandoned in place. +// +// FAILED_PREPARING used to be terminal, so the instance's Deployments, Service +// and ConfigMaps kept running and its Gateway member stayed PREPARING - which +// the Gateway counts as LIVE. One such candidate at desired+surge refused every +// later registration: no repair, no rollout, and a whole leaked Trino cluster. +func TestFailedCandidateIsCleanedUpAndReleasesItsSlot(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Spec.DesiredInstances, harness.operator.config.Spec.MinServing = 1, 1 + harness.store.pool.DesiredInstances, harness.store.pool.MinServing = 1, 1 + + // create -> CREATING -> PREPARING (registered) + harness.tick(t, 3) + instanceID := harness.store.order[0] + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhasePreparing) { + t.Fatalf("instance phase = %s, want PREPARING", phase) + } + // The Gateway observes the coordinator identity itself at registration, and + // a later loss claim has to present exactly what it recorded. + if harness.store.instances[instanceID].CoordinatorID == "" { + t.Fatal("the coordinator identity the Gateway recorded was not kept; a loss claim can never be accepted") + } + + // The coordinator restarts before admission: the registered incarnation is + // gone, so this candidate can never be admitted. + harness.operator.validate = func(context.Context, string, trinoPoolObservation, trinoPoolExpectation) (trinoPoolValidation, error) { + return trinoPoolValidation{ + NodeID: "node-1", ProcessID: "process-restarted", CoordinatorID: "abcde", + AppliedRevision: 42, AuthRevision: "auth", ReadyWorkers: 4, + Checks: []string{trinoPoolCheckImage}, CertificateHash: "hash", + }, nil + } + harness.tick(t, 1) + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseFailedPreparing) { + t.Fatalf("instance phase = %s, want FAILED_PREPARING", phase) + } + + // Nothing may be declared lost while the pods are still there. + harness.kube.absent = false + harness.tick(t, 1) + for _, call := range harness.gateway.calls { + if call == "lost:"+instanceID { + t.Fatal("a loss was claimed while the resources were still present") + } + } + + harness.kube.absent = true + harness.tick(t, 3) + if !harness.kube.deleted[instanceID] { + t.Fatal("the failed candidate's Kubernetes objects were never deleted") + } + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseFailureRetired) { + t.Fatalf("instance phase = %s, want FAILURE_RETIRED", phase) + } + // The Gateway's own never-admitted retirement is what releases the slot: a + // PREPARING member that admitted no work may be retired directly, and the + // Gateway verifies that rather than taking this controller's word for it. + if member, _ := harness.gateway.GetMember(context.Background(), "cell-001", instanceID); member.Phase != "RETIRED" { + t.Fatalf("gateway member phase = %s, want RETIRED so the live slot is released", member.Phase) + } + + // With the slot released the pool replaces the failed candidate instead of + // stalling behind it. + harness.tick(t, 1) + if len(harness.store.order) != 2 { + t.Fatalf("the pool created %d instances; a failed candidate blocked the replacement", len(harness.store.order)) + } +} + +// The staleness hazard is a process that resolved its configuration at boot and +// only later won the lease: publishing that snapshot is a legal fenced write of +// old content, and no generation ordering catches it, because a settings-only +// edit need not move the generation at all. The desired state is therefore +// re-read immediately before it is published. +func TestDesiredStateIsResolvedOnEveryTick(t *testing.T) { + harness := newOperatorHarness(t) + current := harness.operator.config + harness.operator.resolveConfig = func() (trinoPoolConfig, error) { return current, nil } + + harness.tick(t, 1) + if harness.store.pool.DesiredInstances != 3 { + t.Fatalf("desired instances = %d, want the resolved 3", harness.store.pool.DesiredInstances) + } + + // The cluster's configuration changes with no change to the generation, + // which is exactly the case an ordering check cannot see. + changed := current + changed.Spec.DesiredInstances, changed.Spec.MinServing = 5, 4 + current = changed + + harness.tick(t, 1) + if harness.store.pool.DesiredInstances != 5 || harness.store.pool.MinServing != 4 { + t.Fatalf("desired = %d/%d, want the configuration the source holds now (5/4)", + harness.store.pool.DesiredInstances, harness.store.pool.MinServing) + } +} + +// An unreadable configuration holds the last-good state. It is never a desired +// count of zero and never a failed tick that stops the loop: the pool keeps +// serving while somebody fixes the mount. +func TestUnreadableConfigurationFreezesRatherThanEmptiesThePool(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.resolveConfig = func() (trinoPoolConfig, error) { + return trinoPoolConfig{}, errors.New("blueprint is unreadable") + } + + harness.tick(t, 2) + + if harness.store.pool.DesiredInstances != 3 { + t.Fatalf("desired instances = %d, want the last-good 3", harness.store.pool.DesiredInstances) + } + if !harness.store.pool.Frozen { + t.Fatal("an unreadable configuration did not freeze the pool") + } + if len(harness.kube.applied) != 0 { + t.Fatalf("the frozen pool created %d instances", len(harness.kube.applied)) + } +} + +// A generation that went backwards is a configuration problem, not a lost +// fence. Ending the leadership term over it handed the pool to a replica +// reading the same file, which did the same thing: the pool never converged and +// the epoch ratcheted on every tick. +func TestBackwardsGenerationFreezesAndKeepsTheLease(t *testing.T) { + harness := newOperatorHarness(t) + harness.store.staleGeneration = true + + harness.tick(t, 1) + + if !harness.store.pool.Frozen { + t.Fatal("a backwards desired generation did not freeze the pool") + } + if harness.operator.fenced { + t.Fatal("a stale generation was treated as a lost fence") + } + if harness.operator.lease.Epoch == 0 { + t.Fatal("the leadership term ended over a configuration problem") + } +} + +// Leadership can move to a replica whose own copy of the configuration is old. +// Desired state must still be the cluster's, so the publication is derived from +// the API object at the moment of the write - by both replicas, in either +// order. +func TestLeadershipSwitchPublishesTheAPIObjectNotTheReplicaSnapshot(t *testing.T) { + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + t.Setenv(envTrinoCellsFile, mountedRegistry(t, 3, 3)) + client := poolConfigMap(t, 5, 4) + reader := poolAPIReader(t, client) + + // One durable pool, two control planes. Each booted with a different copy + // of the configuration, which is what independent projected volumes look + // like in practice. + shared := newOperatorHarness(t) + stale := newOperatorHarness(t) + stale.operator.store = shared.store + stale.store = shared.store + stale.operator.config.Spec.DesiredInstances, stale.operator.config.Spec.MinServing = 3, 3 + shared.operator.config.Spec.DesiredInstances, shared.operator.config.Spec.MinServing = 2, 2 + for _, harness := range []*operatorHarness{shared, stale} { + harness.operator.resolveConfig = func() (trinoPoolConfig, error) { + return resolveTrinoPoolConfigByID(context.Background(), reader, "cell-001") + } + } + + shared.tick(t, 1) + if shared.store.pool.DesiredInstances != 5 || shared.store.pool.MinServing != 4 { + t.Fatalf("first leader published %d/%d, want the API object's 5/4", + shared.store.pool.DesiredInstances, shared.store.pool.MinServing) + } + + // The lease moves. The new leader's own snapshot says 3/3 and must not + // revert the pool to it. + stale.tick(t, 1) + if shared.store.pool.DesiredInstances != 5 || shared.store.pool.MinServing != 4 { + t.Fatalf("the new leader reverted desired state to %d/%d", + shared.store.pool.DesiredInstances, shared.store.pool.MinServing) + } + + // And a change made while the first leader is idle is picked up by whoever + // is leading, without a restart. + setPoolConfigMap(t, client, 4, 3) + stale.tick(t, 1) + if shared.store.pool.DesiredInstances != 4 || shared.store.pool.MinServing != 3 { + t.Fatalf("desired = %d/%d, want the updated 4/3", + shared.store.pool.DesiredInstances, shared.store.pool.MinServing) + } +} + +// --------------------------------------------------------------------------- +// Publication barrier fake. +// +// This models the rules PoolStore actually enforces, because the operator's +// decisions are only correct against those rules and a permissive fake proves +// the operator can drive a protocol nobody implements. Specifically: +// +// - the operation/step JOURNAL: one row per (operationId, stepId); an +// identical body resolves to the recorded result and APPLIES NOTHING, a +// different body under the same identity is POOL_INTENT_CHANGED forever, +// and a step is recorded only when the effect succeeded; +// - the membership generation CAS on open and commit, and the serving floor; +// - one open publication per tenant, and an immutable plan per publication id; +// - receipts bound to the member's exact (podUid, bootId) and to the +// publication's target revision, accepted only while it is OPEN; +// - missing members computed from the CURRENT active membership, not from the +// list frozen when the barrier opened; +// - a joining member must acknowledge the open publication's target revision, +// which is what makes an open barrier block admission; +// - revocation keeps the tenant's principal rows (PoolStore only rewrites the +// admission row and abandons that tenant's open publications). +// --------------------------------------------------------------------------- + +type fakePublication struct { + trinogateway.Publication + received map[string]fakeReceipt + opened int64 +} + +// fakeReceipt is one row of pool_publication_receipt: what a member said it was +// serving, bound to the process that said it. +type fakeReceipt struct { + bootID string + fingerprint string +} + +// fakeStep is one row of the Gateway's pool_operation journal. +type fakeStep struct { + payload string +} + +// guardStep mirrors PoolStore.inPool's replay resolution. It returns true when +// the step was already recorded, in which case the caller must apply nothing. +func (f *fakePoolGateway) guardStep(step trinogateway.Step, intent string) (bool, error) { + if f.journal == nil { + f.journal = map[string]fakeStep{} + } + if step.OperationID == "" || step.StepID == "" { + return false, fmt.Errorf("%w: a step needs an operation and a step id", trinogateway.ErrValidation) + } + if len(step.StepID) > 64 { + // pool_operation.step_id is VARCHAR(64). + return false, fmt.Errorf("%w: step id %q exceeds 64 characters", trinogateway.ErrValidation, step.StepID) + } + recorded, found := f.journal[step.OperationID+"\x00"+step.StepID] + if !found { + return false, nil + } + if recorded.payload != intent { + f.intentConflicts++ + return false, fmt.Errorf("%w: step %s of %s was recorded with a different intent", + trinogateway.ErrIntentChanged, step.StepID, step.OperationID) + } + return true, nil +} + +// recordStep is the journal write PoolStore performs after a successful effect. +func (f *fakePoolGateway) recordStep(step trinogateway.Step, intent string) { + f.journal[step.OperationID+"\x00"+step.StepID] = fakeStep{payload: intent} +} + +// oldestOpenPublication is PoolStore.openPublicationRow: the pool's oldest OPEN +// publication, which is the one a joining member is measured against. +func (f *fakePoolGateway) oldestOpenPublication() *fakePublication { + var oldest *fakePublication + for _, publication := range f.publications { + if publication.Phase != "OPEN" { + continue + } + if oldest == nil || publication.opened < oldest.opened { + oldest = publication + } + } + return oldest +} + +func (f *fakePoolGateway) GetPool(context.Context, string) (trinogateway.PoolState, error) { + serving := int64(0) + for _, member := range f.members { + if member.Phase == "ACTIVE" { + serving++ + } + } + return trinogateway.PoolState{ + ServingMembers: serving, + MembershipGeneration: f.membership, + }, nil +} + +func (f *fakePoolGateway) activeInstanceIDs() []string { + var active []string + for id, member := range f.members { + if member.Phase == "ACTIVE" { + active = append(active, id) + } + } + sort.Strings(active) + return active +} + +func (f *fakePoolGateway) OpenPublication(_ context.Context, _ string, request trinogateway.OpenPublicationRequest) (trinogateway.Publication, error) { + f.record("open:" + request.Tenant) + if f.publications == nil { + f.publications = map[string]*fakePublication{} + } + intent := strings.Join([]string{request.PublicationID, request.Tenant, request.TargetRevision, + request.PayloadHash, fmt.Sprint(request.ExpectedMembershipGeneration)}, "|") + replayed, err := f.guardStep(request.Step, intent) + if err != nil { + return trinogateway.Publication{}, err + } + if replayed { + return f.publicationView(f.publications[request.PublicationID]), nil + } + if request.ExpectedMembershipGeneration != f.membership { + return trinogateway.Publication{}, fmt.Errorf("%w: the pool membership generation changed", trinogateway.ErrMembershipChanged) + } + active := f.activeInstanceIDs() + if int64(len(active)) < f.minServing { + return trinogateway.Publication{}, fmt.Errorf("%w: a publication requires the minimum serving membership", trinogateway.ErrServingFloor) + } + if existing, found := f.publications[request.PublicationID]; found { + // The plan of a publication identity is immutable. + if existing.Tenant != request.Tenant || existing.TargetRevision != request.TargetRevision { + return trinogateway.Publication{}, fmt.Errorf("%w: this publication identity has a different plan", trinogateway.ErrIntentChanged) + } + f.recordStep(request.Step, intent) + return f.publicationView(existing), nil + } + for _, publication := range f.publications { + if publication.Tenant == request.Tenant && publication.Phase == "OPEN" { + return trinogateway.Publication{}, fmt.Errorf("%w: this tenant already has an open publication", trinogateway.ErrPublicationBarrier) + } + } + f.clock++ + publication := &fakePublication{ + Publication: trinogateway.Publication{ + PublicationID: request.PublicationID, + Tenant: request.Tenant, + TargetRevision: request.TargetRevision, + MembershipGeneration: request.ExpectedMembershipGeneration, + Phase: "OPEN", + RequiredMembers: active, + TenantState: "PENDING", + }, + received: map[string]fakeReceipt{}, + opened: f.clock, + } + f.publications[request.PublicationID] = publication + f.recordStep(request.Step, intent) + return f.publicationView(publication), nil +} + +func (f *fakePoolGateway) AbandonPublication(_ context.Context, _, publicationID string, step trinogateway.Step) (trinogateway.Publication, error) { + f.record("abandon:" + publicationID) + publication, found := f.publications[publicationID] + if !found { + return trinogateway.Publication{}, fmt.Errorf("%w: %s", trinogateway.ErrNotFound, publicationID) + } + replayed, err := f.guardStep(step, "abandon|"+publicationID) + if err != nil { + return trinogateway.Publication{}, err + } + if replayed { + return f.publicationView(publication), nil + } + // An admitted gate is never retracted by abandoning it: the Gateway refuses, + // and the caller reads it back as ADMITTED. + if publication.Phase == "ADMITTED" { + return trinogateway.Publication{}, fmt.Errorf("%w: a committed publication cannot be abandoned", trinogateway.ErrIrreversible) + } + publication.Phase = "ABANDONED" + f.recordStep(step, "abandon|"+publicationID) + return f.publicationView(publication), nil +} + +func (f *fakePoolGateway) GetPublication(_ context.Context, _, publicationID string) (trinogateway.Publication, error) { + publication, found := f.publications[publicationID] + if !found { + return trinogateway.Publication{}, fmt.Errorf("%w: %s", trinogateway.ErrNotFound, publicationID) + } + return f.publicationView(publication), nil +} + +func (f *fakePoolGateway) RecordPublicationReceipt(_ context.Context, _, publicationID string, request trinogateway.PublicationReceiptRequest) (trinogateway.Publication, error) { + f.record("receipt:" + request.InstanceID) + publication, found := f.publications[publicationID] + if !found { + return trinogateway.Publication{}, fmt.Errorf("%w: %s", trinogateway.ErrNotFound, publicationID) + } + intent := strings.Join([]string{publicationID, request.InstanceID, request.PodUID, + request.BootID, request.AppliedRevision, request.AuthFingerprint}, "|") + replayed, err := f.guardStep(request.Step, intent) + if err != nil { + return trinogateway.Publication{}, err + } + if replayed { + return f.publicationView(publication), nil + } + if publication.Phase != "OPEN" { + return trinogateway.Publication{}, fmt.Errorf("%w: only an open publication accepts receipts", trinogateway.ErrPhase) + } + if request.AppliedRevision != publication.TargetRevision { + return trinogateway.Publication{}, fmt.Errorf("%w: applied revision does not match the target", trinogateway.ErrPublicationBarrier) + } + member := f.members[request.InstanceID] + if member == nil || member.BootID != request.BootID || member.PodUID != request.PodUID { + return trinogateway.Publication{}, fmt.Errorf("%w: the acknowledgement does not identify this member's process", trinogateway.ErrPublicationBarrier) + } + if member.Phase != "ACTIVE" && member.Phase != "PREPARING" { + return trinogateway.Publication{}, fmt.Errorf("%w: a %s member cannot acknowledge a publication", trinogateway.ErrPhase, member.Phase) + } + publication.received[request.InstanceID] = fakeReceipt{bootID: request.BootID, fingerprint: request.AuthFingerprint} + f.recordStep(request.Step, intent) + return f.publicationView(publication), nil +} + +func (f *fakePoolGateway) CommitPublication(_ context.Context, _, publicationID string, request trinogateway.CommitPublicationRequest) (trinogateway.Publication, error) { + f.record("commit:" + publicationID) + publication, found := f.publications[publicationID] + if !found { + return trinogateway.Publication{}, fmt.Errorf("%w: %s", trinogateway.ErrNotFound, publicationID) + } + intent := fmt.Sprintf("commit|%s|%d", publicationID, request.ExpectedMembershipGeneration) + replayed, err := f.guardStep(request.Step, intent) + if err != nil { + return trinogateway.Publication{}, err + } + if replayed { + return f.publicationView(publication), nil + } + if publication.Phase == "ADMITTED" { + return f.publicationView(publication), nil + } + if publication.Phase != "OPEN" { + return trinogateway.Publication{}, fmt.Errorf("%w: an abandoned publication cannot be committed", trinogateway.ErrPhase) + } + // Both generations are checked, exactly as PoolStore.commitPublication does: + // the caller's view of the pool AND the membership this barrier was opened + // against. + if f.membership != request.ExpectedMembershipGeneration || publication.MembershipGeneration != request.ExpectedMembershipGeneration { + return trinogateway.Publication{}, fmt.Errorf("%w: the membership generation changed during the publication", trinogateway.ErrMembershipChanged) + } + if int64(len(f.activeInstanceIDs())) < f.minServing { + return trinogateway.Publication{}, fmt.Errorf("%w: the admitting membership fell below the minimum serving count", trinogateway.ErrServingFloor) + } + view := f.publicationView(publication) + if len(view.MissingMembers) > 0 { + return trinogateway.Publication{}, fmt.Errorf("%w: %v", trinogateway.ErrReceiptsIncomplete, view.MissingMembers) + } + publication.Phase, publication.TenantState = "ADMITTED", "ADMITTED" + publication.AdmittedRevision = publication.TargetRevision + if f.admitted == nil { + f.admitted = map[string]string{} + } + f.admitted[publication.Tenant] = publication.TargetRevision + // pool_tenant_admission holds ONE state per tenant: a commit moves a + // revoked tenant back to ADMITTED. + delete(f.revoked, publication.Tenant) + f.recordStep(request.Step, intent) + return f.publicationView(publication), nil +} + +func (f *fakePoolGateway) RevokeTenant(_ context.Context, _, tenant string, request trinogateway.RevokeTenantRequest) (trinogateway.TenantAdmission, error) { + f.record("revoke:" + tenant) + if request.Reason == "" { + return trinogateway.TenantAdmission{}, fmt.Errorf("%w: a revocation must carry a reason", trinogateway.ErrValidation) + } + if f.deferRevoke[tenant] { + delete(f.deferRevoke, tenant) + captured := request + f.deferred = append(f.deferred, func() { _, _ = f.applyRevoke(tenant, captured) }) + return trinogateway.TenantAdmission{}, fmt.Errorf("%w: the response was lost", trinogateway.ErrUnavailable) + } + return f.applyRevoke(tenant, request) +} + +func (f *fakePoolGateway) applyRevoke(tenant string, request trinogateway.RevokeTenantRequest) (trinogateway.TenantAdmission, error) { + intent := "revoke|" + tenant + "|" + request.Reason + replayed, err := f.guardStep(request.Step, intent) + if err != nil { + return trinogateway.TenantAdmission{}, err + } + if replayed { + return trinogateway.TenantAdmission{Tenant: tenant, State: "REVOKED"}, nil + } + delete(f.admitted, tenant) + // PoolStore.revokeTenant rewrites the ADMISSION row and abandons the + // tenant's open publications. It does NOT delete pool_tenant_principal, so + // the binding survives a revocation - which is precisely why re-publishing + // an identical set after one is invisible unless it carries a new occurrence. + for _, publication := range f.publications { + if publication.Tenant == tenant && publication.Phase == "OPEN" { + publication.Phase = "ABANDONED" + } + } + if f.revoked == nil { + f.revoked = map[string]bool{} + } + f.revoked[tenant] = true + f.recordStep(request.Step, intent) + return trinogateway.TenantAdmission{Tenant: tenant, State: "REVOKED"}, nil +} + +// publicationView mirrors PoolStore.publication: requiredMembers is the list +// frozen when the barrier opened, but missingMembers is recomputed against the +// CURRENT active membership and each receipt's recorded process identity. +func (f *fakePoolGateway) publicationView(publication *fakePublication) trinogateway.Publication { + view := publication.Publication + view.Receipts = nil + view.MissingMembers = nil + for _, instanceID := range f.activeInstanceIDs() { + member := f.members[instanceID] + if receipt, acknowledged := publication.received[instanceID]; acknowledged && receipt.bootID == member.BootID { + view.Receipts = append(view.Receipts, trinogateway.PublicationReceipt{ + InstanceID: instanceID, BootID: receipt.bootID, AppliedRevision: publication.TargetRevision, + AuthFingerprint: receipt.fingerprint, + }) + continue + } + view.MissingMembers = append(view.MissingMembers, instanceID) + } + return view +} + +// fakePublicationStore is the durable publication record. It is a map, but the +// operator must treat it as the only source of "already published" - not its +// own memory - so the tests below restart the operator and assert that. +type fakePublicationStore struct { + rows map[string]*configstore.TrinoPoolPublication + // failPrincipalCheckpoint models the half this control plane owns failing + // on its own: the Gateway answered, the local write did not land. + failPrincipalCheckpoint bool +} + +func newFakePublicationStore() *fakePublicationStore { + return &fakePublicationStore{rows: map[string]*configstore.TrinoPoolPublication{}} +} + +func (f *fakePublicationStore) ListTrinoPoolPublications(_ context.Context, poolID string) ([]configstore.TrinoPoolPublication, error) { + ids := make([]string, 0, len(f.rows)) + for id := range f.rows { + ids = append(ids, id) + } + sort.Strings(ids) + publications := make([]configstore.TrinoPoolPublication, 0, len(ids)) + for _, id := range ids { + if f.rows[id].PoolID == poolID { + publications = append(publications, *f.rows[id]) + } + } + return publications, nil +} + +func (f *fakePublicationStore) row(poolID, orgID string) *configstore.TrinoPoolPublication { + if f.rows[orgID] == nil { + f.rows[orgID] = &configstore.TrinoPoolPublication{PoolID: poolID, OrgID: orgID} + } + return f.rows[orgID] +} + +func (f *fakePublicationStore) RecordTrinoPoolTenantPrincipals(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID, revision string) error { + if f.failPrincipalCheckpoint { + return errors.New("the checkpoint did not land") + } + row := f.row(poolID, orgID) + row.PrincipalRevision = revision + if row.State == configstore.TrinoPublicationRevoked || row.State == "" { + row.State = configstore.TrinoPublicationPublished + } + // The Gateway answered: the occurrence is spent. + row.PendingIntent, row.PendingPayload = "", "{}" + return nil +} + +// RecordTrinoPoolPublicationOpen mirrors the real statement: it records the +// LIVE barrier, and moves the state only for a tenant that has never been +// admitted. A tenant that is admitted today stays admitted while the barrier +// for its newest login runs. +func (f *fakePublicationStore) RecordTrinoPoolPublicationOpen(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID, publicationID, target string) error { + row := f.row(poolID, orgID) + row.PublicationID, row.TargetRevision = publicationID, target + if row.AdmittedTargetRevision == "" { + row.State = configstore.TrinoPublicationAdmitting + } + return nil +} + +func (f *fakePublicationStore) RecordTrinoPoolPublicationCommitted(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID, target, receipt string) error { + row := f.row(poolID, orgID) + row.AdmittedTargetRevision, row.GatewayReceipt = target, receipt + row.State = configstore.TrinoPublicationAdmitted + // The barrier is finished; its outcome lives in the admitted revision. + row.PublicationID, row.TargetRevision, row.LastError = "", "", "" + return nil +} + +// ClearTrinoPoolPublicationBarrier mirrors the durable half of abandoning an +// attempt: the barrier pointer goes, the tenant's admission does not. +func (f *fakePublicationStore) ClearTrinoPoolPublicationBarrier(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID string) error { + row := f.row(poolID, orgID) + row.PublicationID, row.TargetRevision = "", "" + switch { + case row.AdmittedTargetRevision != "": + case row.PrincipalRevision != "": + row.State = configstore.TrinoPublicationPublished + default: + row.State = configstore.TrinoPublicationPending + } + return nil +} + +func (f *fakePublicationStore) RecordTrinoPoolTenantRevoked(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID, reason string) error { + row := f.row(poolID, orgID) + row.State, row.LastError = configstore.TrinoPublicationRevoked, reason + row.AdmittedTargetRevision, row.TargetRevision, row.PublicationID = "", "", "" + row.PendingIntent, row.PendingPayload = "", "{}" + return nil +} + +func (f *fakePublicationStore) BeginTrinoPoolPublicationAttempt(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID string) (int64, error) { + row := f.row(poolID, orgID) + row.Attempt++ + return row.Attempt, nil +} + +func (f *fakePublicationStore) BeginTrinoPoolPublicationIntent(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID, kind, payload string) (int64, error) { + if payload == "" { + return 0, errors.New("a publication intent requires its request body") + } + row := f.row(poolID, orgID) + row.Attempt++ + row.PendingIntent = kind + row.PendingPayload = payload + return row.Attempt, nil +} + +func (f *fakePublicationStore) ResolveTrinoPoolPublicationIntent(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID string) error { + row := f.row(poolID, orgID) + row.PendingIntent, row.PendingPayload = "", "{}" + return nil +} + +func (f *fakePublicationStore) RecordTrinoPoolPublicationFailure(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID string, nextAttemptAt time.Time, lastError string) error { + row := f.row(poolID, orgID) + row.Attempts++ + next := nextAttemptAt + row.NextAttemptAt, row.LastError = &next, lastError + return nil +} + +func (f *fakePublicationStore) ClearTrinoPoolPublicationFailure(_ context.Context, _ configstore.TrinoPoolLease, poolID, orgID string) error { + row := f.row(poolID, orgID) + row.Attempts, row.NextAttemptAt, row.LastError = 0, nil, "" + return nil +} + +// servingPool brings the pool to a state where every instance is ACTIVE and +// serving, which is what a publication barrier requires. +func (h *operatorHarness) servingPool(t *testing.T) { + t.Helper() + h.tick(t, 12) + for id, instance := range h.store.instances { + if instance.Phase != string(trinopool.PhaseServing) { + t.Fatalf("instance %s is %s, want SERVING before a publication", id, instance.Phase) + } + } +} + +// Publishing a tenant's principals does NOT admit it. The Gateway dispatches +// work only for a tenant in state ADMITTED, and only a committed barrier puts +// it there - so a gate enabled without this driver denies every tenant forever. +func TestTenantIsAdmittedOnlyThroughACommittedBarrier(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.servingPool(t) + + // Principals first, then the barrier: open, one receipt per member, commit. + harness.tick(t, 10) + + if harness.gateway.admitted["org-a"] == "" { + t.Fatalf("the tenant was never admitted; gateway calls: %v", harness.gateway.calls) + } + row := harness.publications.rows["org-a"] + if row == nil || row.State != configstore.TrinoPublicationAdmitted { + t.Fatalf("durable publication = %+v, want an admitted tenant", row) + } + if row.AdmittedTargetRevision == "" || row.AdmittedTargetRevision != harness.gateway.admitted["org-a"] { + t.Fatalf("durable target %q disagrees with the Gateway's %q", + row.AdmittedTargetRevision, harness.gateway.admitted["org-a"]) + } + // Every serving member had to acknowledge, one receipt each. + receipts := 0 + for _, call := range harness.gateway.calls { + if strings.HasPrefix(call, "receipt:") { + receipts++ + } + } + if receipts != len(harness.store.instances) { + t.Fatalf("%d receipts recorded for %d members", receipts, len(harness.store.instances)) + } +} + +// A member that is not serving the tenant's configuration yet must not be +// acknowledged on its behalf. Committing without it would admit the tenant to a +// coordinator that cannot resolve its catalog or authenticate its login. +func TestBarrierWaitsForAMemberThatIsNotCurrent(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.acknowledgement = func(context.Context, string, trinoPoolProjectionRevisions, int64) (trinoPoolAcknowledgement, error) { + return trinoPoolAcknowledgement{ProcessID: "process-1", AppliedRevision: 42, ProjectionCurrent: false}, nil + } + harness.servingPool(t) + + harness.tick(t, 10) + + if harness.gateway.admitted["org-a"] != "" { + t.Fatal("a tenant was admitted while a member was not serving its configuration") + } + for _, call := range harness.gateway.calls { + if strings.HasPrefix(call, "receipt:") { + t.Fatalf("a receipt was recorded for a member that is not current: %v", harness.gateway.calls) + } + } +} + +// "Already published" cannot live in the leader's memory. A new leadership term +// - or a different replica - reads the durable record, so an admitted tenant is +// not republished and an unpublished one is not skipped. +func TestPublicationStateSurvivesALeadershipChange(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.servingPool(t) + harness.tick(t, 10) + if harness.gateway.admitted["org-a"] == "" { + t.Fatalf("setup: the tenant was never admitted; calls: %v", harness.gateway.calls) + } + + // A different control plane takes over: same durable state, no memory. + successor := newOperatorHarness(t) + successor.operator.store = harness.store + successor.operator.publications = harness.publications + successor.operator.gateway = harness.gateway + successor.operator.config.Pool.TenantAdmission = true + successor.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.gateway.calls = nil + + successor.tick(t, 3) + + for _, call := range harness.gateway.calls { + if strings.HasPrefix(call, "principals:") || strings.HasPrefix(call, "open:") || strings.HasPrefix(call, "commit:") { + t.Fatalf("the new leader republished an already admitted tenant: %v", harness.gateway.calls) + } + } +} + +// A tenant that disappears from the projection is REVOKED, not forgotten. The +// Gateway replaces a principal set only when it is published, so a removed +// warehouse would otherwise keep its logins dispatchable indefinitely. +func TestDepartedTenantIsRevoked(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.servingPool(t) + harness.tick(t, 10) + if harness.gateway.admitted["org-a"] == "" { + t.Fatalf("setup: the tenant was never admitted; calls: %v", harness.gateway.calls) + } + + tenants.orgs = nil + harness.tick(t, 2) + + if !harness.gateway.revoked["org-a"] { + t.Fatalf("the departed tenant was not revoked: %v", harness.gateway.calls) + } + if row := harness.publications.rows["org-a"]; row == nil || row.State != configstore.TrinoPublicationRevoked { + t.Fatalf("durable publication = %+v, want a revoked tenant", row) + } + + // And it is revoked exactly once: the row is kept precisely so the next + // tick does not repeat an external mutation. + harness.gateway.calls = nil + harness.tick(t, 2) + for _, call := range harness.gateway.calls { + if strings.HasPrefix(call, "revoke:") { + t.Fatalf("the tenant was revoked again: %v", harness.gateway.calls) + } + } +} + +// A failing external call earns a wait, and the wait is DURABLE. Keeping it in +// the leader's memory would let a restart - or a leadership move - retry +// immediately, turning a persistent failure into a hot loop against the +// Gateway. A successful one closes its operation, so the table does not grow +// without bound and work in flight stays distinguishable from work that ended. +func TestFailedStepEarnsADurableWaitAndSuccessClosesTheOperation(t *testing.T) { + harness := newOperatorHarness(t) + operations := newFakeOperationStore(func() int64 { return harness.store.epoch }) + harness.operator.operations = operations + + refusal := &trinogateway.Error{Code: "POOL_NOT_CERTIFIED", Status: 409} + harness.gateway.admitErr = refusal + harness.tickTolerant(6) + + operation, known := operations.operations["instance:"+harness.store.order[0]] + if !known { + t.Fatalf("no operation was recorded: %v", operations.operations) + } + if operation.Attempts == 0 || operation.NextAttemptAt == nil { + t.Fatalf("operation = %+v, want a recorded attempt and a next attempt time", operation) + } + if !operation.NextAttemptAt.After(time.Now().UTC()) { + t.Fatalf("next attempt %s is not in the future", operation.NextAttemptAt) + } + if operation.TerminalAt != nil { + t.Fatal("a failed attempt closed the operation; it must stay open to be retried") + } + + // The wait is honoured rather than retried on the next tick. The count is + // per INSTANCE: the backoff is recorded against this instance's operation, + // and its siblings keep making their own attempts - one member's failure + // does not stop the rest of the pool. + waiting := "admit:" + harness.store.order[0] + attempts := countGatewayCalls(harness.gateway.calls, waiting) + harness.tickTolerant(3) + if countGatewayCalls(harness.gateway.calls, waiting) != attempts { + t.Fatalf("the admission was retried during its recorded wait: %v", harness.gateway.calls) + } + + // Once the wait elapses and the call succeeds, the operation is closed. + harness.gateway.admitErr = nil + past := time.Now().UTC().Add(-time.Minute) + stored := operations.operations["instance:"+harness.store.order[0]] + stored.NextAttemptAt = &past + operations.operations["instance:"+harness.store.order[0]] = stored + + harness.tickTolerant(3) + closed := operations.operations["instance:"+harness.store.order[0]] + if closed.TerminalAt == nil || closed.Phase != "completed" { + t.Fatalf("operation = %+v, want a completed, terminal operation", closed) + } +} + +func countGatewayCalls(calls []string, prefix string) int { + count := 0 + for _, call := range calls { + if strings.HasPrefix(call, prefix) { + count++ + } + } + return count +} + +// SUSPECT used to have exactly one exit that freed the member's slot: LOST, +// which requires verified absence of every recorded object. A crash-looping +// coordinator keeps its Deployment forever, so it kept its slot forever, and a +// second such failure exhausted the repair budget and stalled the pool. A +// member that cannot be proven dead leaves through the planned drain instead. +func TestSuspectMemberThatCannotBeProvenDeadIsDrained(t *testing.T) { + harness := newOperatorHarness(t) + harness.servingPool(t) + instanceID := harness.store.order[0] + + // The coordinator is crash-looping: unhealthy, but its objects are present, + // so nothing can claim it terminated. + harness.kube.observed.CoordinatorPodUID = "pod-uid-replaced" + harness.kube.absent = false + harness.store.instances[instanceID].PhaseChangedAt = time.Now().Add(-trinoPoolSuspectAfter - time.Minute) + harness.tick(t, 1) + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseSuspect) { + t.Fatalf("instance phase = %s, want SUSPECT", phase) + } + + // It is not dropped the moment it is suspected: suspicion is reversible. + harness.tick(t, 1) + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseSuspect) { + t.Fatalf("instance phase = %s, want it to stay SUSPECT while the grace lasts", phase) + } + + harness.store.instances[instanceID].PhaseChangedAt = time.Now().Add(-trinoPoolSuspectDrainAfter - time.Minute) + harness.tick(t, 1) + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseDraining) { + t.Fatalf("instance phase = %s, want DRAINING so the slot can be released", phase) + } + for _, call := range harness.gateway.calls { + if call == "lost:"+instanceID { + t.Fatal("a loss was claimed for a member whose objects were still present") + } + } +} + +// The Gateway's serving-floor refusal is authoritative even here: a pool at its +// floor keeps a flaky member rather than dropping below it. +func TestSuspectDrainRespectsTheServingFloor(t *testing.T) { + harness := newOperatorHarness(t) + harness.servingPool(t) + instanceID := harness.store.order[0] + harness.kube.observed.CoordinatorPodUID = "pod-uid-replaced" + harness.kube.absent = false + harness.store.instances[instanceID].PhaseChangedAt = time.Now().Add(-trinoPoolSuspectAfter - time.Minute) + harness.tick(t, 1) + + harness.gateway.drainErr = &trinogateway.Error{Code: "POOL_SERVING_FLOOR", Status: 409} + harness.store.instances[instanceID].PhaseChangedAt = time.Now().Add(-trinoPoolSuspectDrainAfter - time.Minute) + harness.tickTolerant(2) + + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseSuspect) { + t.Fatalf("instance phase = %s, want it to stay SUSPECT after a refused drain", phase) + } +} + +// poolOrgs builds a fleet of tenants, each with its own root login. +func poolOrgs(count int) []configstore.TrinoEnabledOrg { + orgs := make([]configstore.TrinoEnabledOrg, 0, count) + for i := 0; i < count; i++ { + orgs = append(orgs, configstore.TrinoEnabledOrg{ + OrgID: fmt.Sprintf("org-%04d", i), + DatabaseName: fmt.Sprintf("acme%04d", i), + CellID: "registered:cell-001", + RootPasswordHash: "hash", + }) + } + return orgs +} + +// admitAll drives the barrier until every tenant is admitted, or gives up. +func (h *operatorHarness) admitAll(t *testing.T, tenants int) { + t.Helper() + for tick := 0; tick < tenants*8+64; tick++ { + h.tickTolerant(1) + admitted := 0 + for _, row := range h.publications.rows { + if row.State == configstore.TrinoPublicationAdmitted { + admitted++ + } + } + if admitted == tenants { + return + } + } + t.Fatalf("not every tenant was admitted after many ticks") +} + +// A new warehouse must not wait behind a re-admission of every existing one. +// +// The driver performs ONE external step per five-second tick, so a target that +// expired for the whole fleet whenever anything changed anywhere would have put +// a new tenant hours behind thousands of pointless re-admissions. A tenant's +// intent is its own: its principals, under its own attempt. +func TestANewTenantDoesNotWaitForTheWholeFleetToBeReadmitted(t *testing.T) { + const existing = 1000 + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: poolOrgs(existing)} + harness.operator.tenants = tenants + harness.servingPool(t) + + // A fleet that is already admitted, as the durable record would hold it + // after those tenants were provisioned. + for _, org := range tenants.orgs { + binding := trinoPoolTenantBindingFor(org) + row := harness.publications.row(harness.operator.config.PoolID, org.OrgID) + row.Attempt = 1 + row.PrincipalRevision = binding.Revision + // An admitted tenant holds no live barrier: the commit cleared it. + row.AdmittedTargetRevision = trinoPoolTargetRevision(binding, row.Attempt) + row.State = configstore.TrinoPublicationAdmitted + } + + // Something changes that moves the pool's catalog revision and the whole + // projection - exactly what provisioning a new warehouse does. + harness.store.pool.PublicationRevision++ + harness.operator.projection = func() trinoPoolProjectionRevisions { + return trinoPoolProjectionRevisions{Policy: "policy-2", Password: "password-2", Group: "group-2"} + } + newcomer := poolOrgs(existing + 1)[existing] + tenants.orgs = append(tenants.orgs, newcomer) + harness.gateway.calls = nil + + // The newcomer is admitted within a handful of steps: publish its binding, + // open, one receipt per serving member, commit. + steps := 0 + for ; steps < 64; steps++ { + harness.tickTolerant(1) + if harness.gateway.admitted[newcomer.OrgID] != "" { + break + } + } + if harness.gateway.admitted[newcomer.OrgID] == "" { + t.Fatalf("the new tenant was not admitted in %d steps", steps) + } + // And nothing re-admitted the existing fleet to get there. + commits := countGatewayCalls(harness.gateway.calls, "commit:") + if commits > 2 { + t.Fatalf("%d publications committed to admit one new tenant; the fleet was being re-admitted", commits) + } +} + +// A tenant whose publication always fails must not hold the queue. The driver +// rotates and honours each tenant's durable backoff, so its neighbours still +// get admitted. +func TestAFailingTenantDoesNotStarveTheOthers(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + orgs := poolOrgs(3) + harness.operator.tenants = &fakeTenantStore{orgs: orgs} + // The first tenant in order can never be published - from the start, so it + // is never admitted and keeps failing. + broken := orgs[0].OrgID + harness.gateway.principalErr = map[string]error{broken: errors.New("principal conflict")} + // The ticks that bring the pool up already carry the failing tenant, which + // is the point: its failure must not stop them either. + harness.tickTolerant(12) + + for tick := 0; tick < 80; tick++ { + harness.tickTolerant(1) + } + + for _, org := range orgs[1:] { + if harness.gateway.admitted[org.OrgID] == "" { + t.Fatalf("tenant %s was starved by the failing tenant %s: %v", org.OrgID, broken, harness.gateway.calls) + } + } + if row := harness.publications.rows[broken]; row == nil || row.Attempts == 0 || row.NextAttemptAt == nil { + t.Fatalf("the failing tenant recorded no durable backoff: %+v", row) + } +} + +// A tenant that is revoked, re-enabled and revoked again needs a NEW durable +// occurrence. A constant step identity would replay the FIRST revocation's +// recorded outcome, leaving a tenant everybody believes is revoked admitted and +// dispatchable. +func TestASecondRevocationIsItsOwnOccurrence(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: poolOrgs(1)} + harness.operator.tenants = tenants + org := tenants.orgs[0].OrgID + harness.servingPool(t) + harness.admitAll(t, 1) + + all := tenants.orgs + tenants.orgs = nil + harness.tickTolerant(3) + if !harness.gateway.revoked[org] { + t.Fatalf("the tenant was not revoked: %v", harness.gateway.calls) + } + firstRevoke := harness.publications.rows[org].Attempt + + // Re-enabled: published and admitted again. + tenants.orgs = all + harness.admitAll(t, 1) + if harness.gateway.admitted[org] == "" { + t.Fatal("the re-enabled tenant was not admitted again") + } + + // Revoked a second time. + tenants.orgs = nil + harness.gateway.revoked = map[string]bool{} + harness.tickTolerant(3) + if !harness.gateway.revoked[org] { + t.Fatalf("the second revocation never happened: %v", harness.gateway.calls) + } + if second := harness.publications.rows[org].Attempt; second <= firstRevoke { + t.Fatalf("the second revocation reused occurrence %d (first was %d)", second, firstRevoke) + } +} + +// Membership changes during a rollout. An attempt opened against the old +// membership can never commit, and while it is open a joining member cannot be +// admitted - the cycle where the commit waits for a replacement the barrier +// itself refuses. The attempt is abandoned and a new one is opened. +func TestBarrierReopensWhenMembershipChanges(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: poolOrgs(1)} + harness.operator.tenants = tenants + org := tenants.orgs[0].OrgID + harness.servingPool(t) + + // Get as far as an open barrier with at least one receipt. + for tick := 0; tick < 8 && countGatewayCalls(harness.gateway.calls, "receipt:") == 0; tick++ { + harness.tickTolerant(1) + } + if countGatewayCalls(harness.gateway.calls, "open:") == 0 { + t.Fatalf("no barrier was opened: %v", harness.gateway.calls) + } + firstAttempt := harness.publications.rows[org].Attempt + + // The membership moves under it, as a replacement does. + harness.gateway.membership++ + harness.gateway.calls = nil + harness.tickTolerant(2) + + if countGatewayCalls(harness.gateway.calls, "abandon:") == 0 { + t.Fatalf("the stale attempt was not abandoned: %v", harness.gateway.calls) + } + if next := harness.publications.rows[org].Attempt; next <= firstAttempt { + t.Fatalf("no new attempt was started (attempt %d, was %d)", next, firstAttempt) + } + + // And it converges: the tenant is admitted against the current membership. + harness.admitAll(t, 1) + if harness.gateway.admitted[org] == "" { + t.Fatalf("the tenant never recovered after the membership change: %v", harness.gateway.calls) + } +} + +// One unserviceable tenant must not stop the pool's compute lifecycle: a +// warehouse that can never be published used to end the tick before any +// instance was repaired, drained or replaced. +func TestAFailingTenantDoesNotBlockInstanceProgress(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: poolOrgs(1)} + harness.gateway.principalErr = map[string]error{"org-0000": errors.New("principal conflict")} + + // The pool still reaches its desired instance count. + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + } + serving := 0 + for _, instance := range harness.store.instances { + if instance.Phase == string(trinopool.PhaseServing) { + serving++ + } + } + if serving != harness.store.pool.DesiredInstances { + t.Fatalf("%d serving instances with a failing tenant, want %d: the tenant blocked the lifecycle", + serving, harness.store.pool.DesiredInstances) + } +} + +// A member joining while a tenant publication is open cannot acknowledge that +// publication's target - it registers under a release id - so the Gateway +// refuses the admission. Without a way out, the member waits for the barrier +// and the barrier waits for the membership that includes the member. The +// barrier gives way. +func TestAnOpenBarrierDoesNotBlockAMemberFromJoining(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: poolOrgs(1)} + // The barrier will open and STAY open: no member is serving the current + // projection yet, so no acknowledgement can be recorded. + harness.operator.acknowledgement = func(context.Context, string, trinoPoolProjectionRevisions, int64) (trinoPoolAcknowledgement, error) { + return trinoPoolAcknowledgement{ProcessID: "process-1", AppliedRevision: 42, ProjectionCurrent: false}, nil + } + harness.servingPool(t) + for tick := 0; tick < 6 && countGatewayCalls(harness.gateway.calls, "open:") == 0; tick++ { + harness.tickTolerant(1) + } + if countGatewayCalls(harness.gateway.calls, "open:") == 0 { + t.Fatalf("no barrier was opened: %v", harness.gateway.calls) + } + + // A replacement instance reaches admission and the Gateway refuses it on + // the barrier. + harness.gateway.admitErr = fmt.Errorf("%w: a member joining during a publication must acknowledge its target revision", + trinogateway.ErrPublicationBarrier) + harness.store.pool.DesiredInstances++ + harness.operator.config.Spec.DesiredInstances++ + harness.gateway.calls = nil + harness.tickTolerant(12) + + if countGatewayCalls(harness.gateway.calls, "abandon:") == 0 { + t.Fatalf("the open barrier was not reopened to let the member in: %v", harness.gateway.calls) + } +} + +// A container can restart inside a Pod and come back ready with the SAME pod +// UID and a NEW Trino process. The Gateway binds the member to the boot +// identity it registered and refuses to dispatch to anything else, so a health +// check that looks only at pod readiness reports SERVING while the pool has +// quietly lost that member's capacity. +func TestARestartedProcessInTheSamePodIsNotTheSameMember(t *testing.T) { + harness := newOperatorHarness(t) + harness.servingPool(t) + instanceID := harness.store.order[0] + instance := harness.store.instances[instanceID] + if instance.CoordinatorBootID == "" { + t.Fatal("setup: the admitted boot identity was not recorded") + } + admitted := instance.CoordinatorBootID + + // Same pod, same UID, ready - and a different process behind it. Clearing + // the pacing map stands in for the probe interval elapsing. + harness.operator.identity = func(context.Context, string) (string, error) { + return "process-restarted", nil + } + harness.operator.identityObservedAt = nil + harness.tick(t, 1) + + if instance.Phase != string(trinopool.PhaseSuspect) { + t.Fatalf("phase = %s, want SUSPECT: the admitted incarnation is gone", instance.Phase) + } + if harness.gateway.members[instanceID].Phase != "SUSPECT" { + t.Fatalf("gateway member = %s, want the member excluded too", harness.gateway.members[instanceID].Phase) + } + // The recorded identity is what the Gateway admitted. Quietly adopting the + // new one would relabel a member nobody certified. + if instance.CoordinatorBootID != admitted { + t.Fatalf("stored boot identity = %q, want the admitted %q left alone", + instance.CoordinatorBootID, admitted) + } + // And it is replaced through the ordinary failure path. + harness.kube.absent = true + harness.kube.observed.PodsPresent = 0 + harness.tick(t, 4) + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseFailureRetired) { + t.Fatalf("phase = %s, want the restarted member retired through the failure path", phase) + } +} + +// A probe that does not answer is not evidence about the member: the +// controller could not look, which is the same as a failed observation. A +// member is never suspected on a timeout. +func TestAnUnansweredIdentityProbeIsNotEvidence(t *testing.T) { + harness := newOperatorHarness(t) + harness.servingPool(t) + instanceID := harness.store.order[0] + + harness.operator.identity = func(context.Context, string) (string, error) { + return "", errors.New("connection refused") + } + harness.operator.identityObservedAt = nil + harness.tick(t, 3) + + if phase := harness.store.instances[instanceID].Phase; phase != string(trinopool.PhaseServing) { + t.Fatalf("phase = %s, want SERVING: an unanswered probe is not evidence", phase) + } + for _, call := range harness.gateway.calls { + if call == "suspect:"+instanceID { + t.Fatal("a member was suspected because its probe timed out") + } + } +} + +// fullMembership ticks until every desired instance is ACTIVE at the Gateway. +// A publication needs the serving floor, which the Gateway enforces on both +// open and commit. +func (h *operatorHarness) fullMembership(t *testing.T) { + t.Helper() + for tick := 0; tick < 120; tick++ { + h.tickTolerant(1) + if len(h.gateway.activeInstanceIDs()) >= h.store.pool.DesiredInstances { + return + } + } + t.Fatalf("the pool never reached %d active members: %v", + h.store.pool.DesiredInstances, h.phases()) +} + +// admitTenantBeyond ticks until the tenant is admitted at a target other than +// the one it already held, and returns the new one. +func (h *operatorHarness) admitTenantBeyond(t *testing.T, tenant, previous string) string { + t.Helper() + for tick := 0; tick < 120; tick++ { + h.tickTolerant(1) + if current := h.gateway.admitted[tenant]; current != "" && current != previous { + return current + } + } + t.Fatalf("tenant %s was never admitted past %q: %v", tenant, previous, h.gateway.calls) + return "" +} + +// waitForPrincipals ticks until the Gateway's binding for a tenant has the +// expected size, and returns what it holds. +func (h *operatorHarness) waitForPrincipals(t *testing.T, tenant string, want int) []string { + t.Helper() + for tick := 0; tick < 120; tick++ { + h.tickTolerant(1) + if len(h.gateway.principals[tenant]) == want { + return h.gateway.principals[tenant] + } + } + t.Fatalf("tenant %s binds %v, want %d principals", tenant, h.gateway.principals[tenant], want) + return nil +} + +// openBarriers is how many publications the Gateway currently holds OPEN. +func (f *fakePoolGateway) openBarriers() int { + open := 0 + for _, publication := range f.publications { + if publication.Phase == "OPEN" { + open++ + } + } + return open +} + +// A tenant needs a SECOND barrier whenever its own logins change, against the +// very same members that acknowledged the first one. +// +// Every durable step identity therefore carries the publication it belongs to. +// Without that, the second attempt's receipt for a member reuses the first +// attempt's step identity while carrying a different applied revision, which +// the Gateway refuses as a changed intent - and refuses forever, because the +// identity never moves again. The tenant is then stranded: published, never +// admitted, retried until somebody notices. +func TestASecondBarrierForOneTenantAcknowledgesTheSameMembers(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + first := harness.gateway.admitted["org-a"] + if first == "" { + t.Fatal("the tenant was not admitted by its first barrier") + } + + // A second login: a new binding, a new barrier, the same members. + tenants.orgs[0].Users = append(tenants.orgs[0].Users, + configstore.TrinoOrgUser{Username: "engineer", PasswordHash: "hash"}) + second := harness.admitTenantBeyond(t, "org-a", first) + if got := harness.gateway.principals["org-a"]; len(got) != 3 { + t.Fatalf("the Gateway binds %v, want the three current logins", got) + } + row := harness.publications.rows["org-a"] + if row.State != configstore.TrinoPublicationAdmitted || row.AdmittedTargetRevision != second { + t.Fatalf("durable publication = %+v, want an admission at %q", row, second) + } + // And the finished attempt is not left behind as the live one. + if row.PublicationID != "" { + t.Fatalf("a committed barrier is still recorded as live: %q", row.PublicationID) + } +} + +// A login that is added and then removed again returns the tenant's principal +// set - and therefore its revision, a digest of that set - to a value it has +// held before. +// +// The publication body is then byte-identical to the earlier one, so under a +// constant step identity the Gateway resolves it as a replay and applies +// NOTHING: the removed login stays bound to the tenant, and the next tenant to +// be given that principal is refused for a conflict nobody can see. Each +// changed binding therefore takes a new durable occurrence first. +func TestAPrincipalSetReturningToAnEarlierValueIsRepublished(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg()}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + tenants.orgs[0].Users = []configstore.TrinoOrgUser{{Username: "analyst", PasswordHash: "hash"}} + harness.waitForPrincipals(t, "org-a", 2) + + tenants.orgs[0].Users = nil + if got := harness.waitForPrincipals(t, "org-a", 1); len(got) != 1 { + t.Fatalf("the removed login is still bound to the tenant: %v", got) + } + // The principal is free again, which is what lets another tenant hold it. + if owner, bound := harness.gateway.principalOf["acme.analyst"]; bound { + t.Fatalf("the removed principal is still owned by %s", owner) + } +} + +// Every OPEN publication refuses a joining member, so leftovers - from an older +// version, or from a leader that died mid-attempt - are an obstacle to the +// pool's own compute lifecycle, not just to their tenants. +// +// The driver releases them ONE per pass, durably, and opens nothing new while a +// candidate is waiting. Releasing without recording it durably was the loop +// that re-abandoned one finished publication on every attempt while the barrier +// actually in the way stayed open. +func TestLeftoverBarriersDoNotBlockAMemberFromJoining(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + orgs := poolOrgs(3) + harness.operator.tenants = &fakeTenantStore{orgs: orgs} + harness.fullMembership(t) + + // Three tenants, each left holding an open barrier: the shape an earlier + // scheduler produced by opening one barrier per eligible tenant. + for index, org := range orgs { + binding := trinoPoolTenantBindingFor(org) + target := trinoPoolTargetRevision(binding, 1) + publicationID := trinoPoolPublicationID(org.OrgID, target) + if _, err := harness.gateway.OpenPublication(context.Background(), "pool", trinogateway.OpenPublicationRequest{ + Step: trinogateway.Step{OperationID: "seed:" + org.OrgID, StepID: fmt.Sprintf("open.%d", index)}, + PublicationID: publicationID, + Tenant: org.OrgID, + TargetRevision: target, + ExpectedMembershipGeneration: harness.gateway.membership, + PayloadHash: trinoPoolPublicationPlanHash(binding, target), + }); err != nil { + t.Fatalf("seeding a leftover barrier for %s: %v", org.OrgID, err) + } + row := harness.publications.row(harness.operator.config.PoolID, org.OrgID) + row.Attempt = 1 + row.PrincipalRevision = binding.Revision + row.PublicationID, row.TargetRevision = publicationID, target + row.State = configstore.TrinoPublicationAdmitting + } + if harness.gateway.openBarriers() != 3 { + t.Fatalf("expected three leftover barriers, got %d", harness.gateway.openBarriers()) + } + + // A replacement instance now has to join. + harness.store.pool.DesiredInstances++ + harness.operator.config.Spec.DesiredInstances++ + joined := false + for tick := 0; tick < 60 && !joined; tick++ { + harness.tickTolerant(1) + serving := 0 + for _, instance := range harness.store.instances { + if trinopool.Phase(instance.Phase) == trinopool.PhaseServing { + serving++ + } + } + joined = serving == harness.store.pool.DesiredInstances + } + if !joined { + t.Fatalf("the joining member never got past the leftover barriers; phases %v, calls %v", + harness.phases(), harness.gateway.calls) + } + // The releases are recorded durably, so no tenant is left naming a + // publication that is finished. + for _, org := range orgs { + row := harness.publications.rows[org.OrgID] + if row.PublicationID != "" && harness.gateway.publications[row.PublicationID].Phase != "OPEN" { + t.Fatalf("tenant %s still names the finished publication %s", org.OrgID, row.PublicationID) + } + } + // And they are still admitted afterwards: releasing an attempt is not + // abandoning the tenant. + harness.admitAll(t, len(orgs)) +} + +// One barrier is live at a time, and it is driven to completion. +// +// This is not tuning. Every open publication blocks every member admission, and +// an attempt that is only advanced once per full rotation of the fleet stays +// open long enough for ordinary membership churn to invalidate it before it can +// commit. +func TestOnlyOneBarrierIsLiveAtATime(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + orgs := poolOrgs(5) + harness.operator.tenants = &fakeTenantStore{orgs: orgs} + harness.fullMembership(t) + + for tick := 0; tick < 120; tick++ { + harness.tickTolerant(1) + if open := harness.gateway.openBarriers(); open > 1 { + t.Fatalf("%d barriers were open at once after %d ticks: %v", + open, tick+1, harness.gateway.calls) + } + } + for _, org := range orgs { + if harness.gateway.admitted[org.OrgID] == "" { + t.Fatalf("tenant %s was never admitted: %v", org.OrgID, harness.gateway.calls) + } + } +} + +// The receipts a commit rests on must describe ONE configuration. +// +// Each receipt is evidence that a member is serving what this control plane +// serves, so an attempt that collects one receipt against the old projection +// and one against the new admits the tenant on a mixture that no member ever +// had. The attempt is released and reopened on the current projection instead - +// and a tenant that is already admitted has no attempt in flight, so it is not +// re-admitted for somebody else's change. +func TestReceiptsRestOnTheProjectionTheAttemptWasOpenedAt(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + projection := trinoPoolProjectionRevisions{Policy: "policy-1", Password: "password-1", Group: "group-1"} + harness.operator.projection = func() trinoPoolProjectionRevisions { return projection } + harness.fullMembership(t) + + // Get the attempt as far as its first receipt. + for tick := 0; tick < 8 && countGatewayCalls(harness.gateway.calls, "receipt:") == 0; tick++ { + harness.tickTolerant(1) + } + if countGatewayCalls(harness.gateway.calls, "receipt:") == 0 { + t.Fatalf("no receipt was recorded: %v", harness.gateway.calls) + } + + // The projection moves under the open attempt. + projection = trinoPoolProjectionRevisions{Policy: "policy-2", Password: "password-2", Group: "group-2"} + harness.admitAll(t, 1) + + admittedID := "" + for id, publication := range harness.gateway.publications { + if publication.Phase == "ADMITTED" { + admittedID = id + } + } + if admittedID == "" { + t.Fatalf("the tenant was never admitted: %v", harness.gateway.calls) + } + expected := trinoPoolProjectionFingerprint(projection) + for instanceID, receipt := range harness.gateway.publications[admittedID].received { + if receipt.fingerprint != expected { + t.Fatalf("member %s acknowledged projection %s, the committed attempt requires %s", + instanceID, receipt.fingerprint, expected) + } + } +} + +// clearBackoff is the durable wait elapsing. Ticks are instantaneous in a test, +// so a tenant that earned a wait would otherwise never be reached again. +func (h *operatorHarness) clearBackoff(tenant string) { + if row := h.publications.rows[tenant]; row != nil { + row.NextAttemptAt = nil + } +} + +// A publication whose response was lost is NOT finished: it may still be +// executing at the Gateway, and it commits whenever it gets there. +// +// If the driver moves on to the next desired binding under a new step +// identity, the two are unordered at the Gateway - same controller, same epoch, +// different steps - so the older one can commit last and leave the tenant bound +// to a set nobody wants, while duckgres has checkpointed the newer one and will +// never publish it again. The step identity is therefore pinned to the tenant's +// open occurrence until that occurrence reaches a DEFINITE outcome, which makes +// the late duplicate a journal replay rather than a second effect. +func TestADelayedPublicationCannotOverwriteANewerBinding(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + // A second login. Its publication reaches the Gateway but the response is + // lost, and the effect is left executing there. + tenants.orgs[0].Users = append(tenants.orgs[0].Users, + configstore.TrinoOrgUser{Username: "engineer", PasswordHash: "hash"}) + harness.gateway.deferPublish = map[string]bool{"org-a": true} + before := countGatewayCalls(harness.gateway.calls, "principals:") + for tick := 0; tick < 12 && countGatewayCalls(harness.gateway.calls, "principals:") == before; tick++ { + harness.tickTolerant(1) + } + if countGatewayCalls(harness.gateway.calls, "principals:") == before { + t.Fatalf("the changed binding was never published: %v", harness.gateway.calls) + } + + // Before that lands, the desired binding changes AGAIN: the first login is + // removed, so the wanted set is neither the admitted one nor the one still + // executing at the Gateway. + tenants.orgs[0].Users = []configstore.TrinoOrgUser{{Username: "engineer", PasswordHash: "hash"}} + wanted := trinoPoolTenantBindingFor(tenants.orgs[0]).Principals + harness.clearBackoff("org-a") + for tick := 0; tick < 8; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + // The delayed effect finally commits, after the newer one. + harness.gateway.deliverDeferred() + for tick := 0; tick < 12; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, wanted) { + t.Fatalf("the Gateway binds %v, want the current %v: a delayed publication overwrote a newer binding", + got, wanted) + } + // And duckgres's checkpoint agrees with what the Gateway actually holds. + row := harness.publications.rows["org-a"] + if row.PrincipalRevision != trinoPoolTenantBindingFor(tenants.orgs[0]).Revision { + t.Fatalf("checkpointed revision %q does not describe the Gateway's binding %v", + row.PrincipalRevision, harness.gateway.principals["org-a"]) + } +} + +// The same hazard, in the direction that silently takes a warehouse off the +// air: a revocation whose response was lost, a tenant that is re-enabled and +// admitted again, and then the old revocation commits. +// +// duckgres would hold an admitted checkpoint for a tenant the Gateway refuses +// to dispatch for, and nothing in the desired state ever changes again to +// correct it. +func TestADelayedRevocationCannotUnadmitAReenabledTenant(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + // The warehouse leaves the projection; its revocation is left executing at + // the Gateway. + all := tenants.orgs + tenants.orgs = nil + harness.gateway.deferRevoke = map[string]bool{"org-a": true} + for tick := 0; tick < 12 && countGatewayCalls(harness.gateway.calls, "revoke:") == 0; tick++ { + harness.tickTolerant(1) + } + if countGatewayCalls(harness.gateway.calls, "revoke:") == 0 { + t.Fatalf("the departed tenant was never revoked: %v", harness.gateway.calls) + } + + // It comes back before that lands. + tenants.orgs = all + harness.clearBackoff("org-a") + for tick := 0; tick < 24; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + // The old revocation finally commits. + harness.gateway.deliverDeferred() + for tick := 0; tick < 24; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + row := harness.publications.rows["org-a"] + if row.State == configstore.TrinoPublicationAdmitted && harness.gateway.revoked["org-a"] { + t.Fatalf("duckgres holds an admitted checkpoint for a tenant the Gateway has revoked: %+v", row) + } + if !harness.gateway.revoked["org-a"] && row.State != configstore.TrinoPublicationAdmitted { + t.Fatalf("the re-enabled tenant is dispatchable at the Gateway but not checkpointed: %+v", row) + } +} + +// The checkpoint is the half of a publication this control plane owns, and it +// can be lost on its own: the Gateway answered, the local write did not land. +// +// The occurrence is still open, so the next pass reissues THAT step rather than +// a new one. The Gateway replays its recorded outcome, and the checkpoint +// catches up - without a second effect and without a new occurrence that an +// older request could still undercut. +func TestALostCheckpointIsSettledUnderTheSameOccurrence(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + tenants.orgs[0].Users = append(tenants.orgs[0].Users, + configstore.TrinoOrgUser{Username: "engineer", PasswordHash: "hash"}) + harness.publications.failPrincipalCheckpoint = true + for tick := 0; tick < 12 && len(harness.gateway.principals["org-a"]) != 3; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + if len(harness.gateway.principals["org-a"]) != 3 { + t.Fatalf("the binding never reached the Gateway: %v", harness.gateway.principals["org-a"]) + } + row := harness.publications.rows["org-a"] + if row.PendingIntent != configstore.TrinoPublicationIntentPrincipals { + t.Fatalf("pending intent = %q, want the occurrence held open by the lost checkpoint", row.PendingIntent) + } + occurrence := row.Attempt + + // The checkpoint works again. The SAME occurrence settles it: the Gateway + // replays, and no second effect is issued. + harness.publications.failPrincipalCheckpoint = false + published := countGatewayCalls(harness.gateway.calls, "principals:") + harness.clearBackoff("org-a") + harness.admitAll(t, 1) + + row = harness.publications.rows["org-a"] + if row.PendingIntent != "" { + t.Fatalf("the occurrence is still open after the checkpoint landed: %+v", row) + } + if row.Attempt != occurrence { + t.Fatalf("occurrence moved from %d to %d to settle a lost checkpoint", occurrence, row.Attempt) + } + if row.PrincipalRevision != trinoPoolTenantBindingFor(tenants.orgs[0]).Revision { + t.Fatalf("checkpoint = %q, want the published binding", row.PrincipalRevision) + } + if again := countGatewayCalls(harness.gateway.calls, "principals:") - published; again != 1 { + t.Fatalf("%d publications were issued to settle a lost checkpoint, want exactly one replay", again) + } +} + +// A leadership move in the middle of an unresolved request changes nothing: the +// occurrence and what it stands for are durable, so the new leader reissues the +// same step rather than minting one the old request could undercut. +func TestANewLeaderSettlesTheOccurrenceItInherits(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + tenants.orgs[0].Users = append(tenants.orgs[0].Users, + configstore.TrinoOrgUser{Username: "engineer", PasswordHash: "hash"}) + harness.gateway.deferPublish = map[string]bool{"org-a": true} + for tick := 0; tick < 12 && harness.publications.rows["org-a"].PendingIntent == ""; tick++ { + harness.tickTolerant(1) + } + row := harness.publications.rows["org-a"] + if row.PendingIntent != configstore.TrinoPublicationIntentPrincipals { + t.Fatalf("pending intent = %q, want an occurrence in flight", row.PendingIntent) + } + occurrence := row.Attempt + + // A new leadership term: a fresh operator over the same durable record. + successor := newOperatorHarness(t) + successor.store, successor.publications = harness.store, harness.publications + successor.gateway = harness.gateway + successor.operator.store = harness.store + successor.operator.publications = harness.publications + successor.operator.gateway = harness.gateway + successor.operator.tenants = tenants + successor.operator.config.Pool.TenantAdmission = true + successor.clearBackoff("org-a") + for tick := 0; tick < 12 && successor.publications.rows["org-a"].PendingIntent != ""; tick++ { + successor.tickTolerant(1) + successor.clearBackoff("org-a") + } + + row = successor.publications.rows["org-a"] + if row.PendingIntent != "" { + t.Fatalf("the new leader left the inherited occurrence open: %+v", row) + } + if row.Attempt != occurrence { + t.Fatalf("the new leader minted occurrence %d instead of settling %d", row.Attempt, occurrence) + } + + // And the original, still executing at the old Gateway connection, cannot + // change anything now. + harness.gateway.deliverDeferred() + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, trinoPoolTenantBindingFor(tenants.orgs[0]).Principals) { + t.Fatalf("the late duplicate changed the binding to %v", got) + } +} + +// The progress bound has to hold at the size the pool is actually for. +// +// A live barrier is finished before any new binding is published, so closing it +// costs a number of passes that depends on the MEMBER count - not on how many +// tenants happen to be waiting for their first publication. Servicing bindings +// first meant a fleet's worth of pending tenants each took a pass before the +// open barrier advanced at all. +func TestManyPendingBindingsDoNotDelayTheLiveBarrier(t *testing.T) { + const tenants = 400 + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + orgs := poolOrgs(tenants) + harness.operator.tenants = &fakeTenantStore{orgs: orgs} + harness.fullMembership(t) + + // One tenant is published and has a barrier open; everybody else is still + // waiting for their first publication. + for tick := 0; tick < 200 && harness.gateway.openBarriers() == 0; tick++ { + harness.tickTolerant(1) + } + if harness.gateway.openBarriers() == 0 { + t.Fatalf("no barrier was opened in a fleet of %d tenants", tenants) + } + var open *fakePublication + for _, publication := range harness.gateway.publications { + if publication.Phase == "OPEN" { + open = publication + } + } + + // One receipt per member, then the commit: the barrier closes within a + // bound set by the membership, with hundreds of bindings still pending. + bound := len(harness.gateway.activeInstanceIDs()) + 3 + for tick := 0; tick < bound && open.Phase == "OPEN"; tick++ { + harness.tickTolerant(1) + } + if open.Phase != "ADMITTED" { + t.Fatalf("the barrier for %s is %s after %d passes with %d tenants pending", + open.Tenant, open.Phase, bound, tenants) + } +} + +// The same bound for the pool's compute: a member waiting to join is not behind +// the fleet's pending bindings either. +func TestManyPendingBindingsDoNotDelayAJoiningMember(t *testing.T) { + const tenants = 400 + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: poolOrgs(tenants)} + harness.fullMembership(t) + for tick := 0; tick < 200 && harness.gateway.openBarriers() == 0; tick++ { + harness.tickTolerant(1) + } + + harness.store.pool.DesiredInstances++ + harness.operator.config.Spec.DesiredInstances++ + joined := false + // Create, observe, register, validate, admit, mark serving: a handful of + // passes, plus one to release the open barrier. Nothing here scales with + // the number of tenants waiting to be published. + for tick := 0; tick < 20 && !joined; tick++ { + harness.tickTolerant(1) + serving := 0 + for _, instance := range harness.store.instances { + if trinopool.Phase(instance.Phase) == trinopool.PhaseServing { + serving++ + } + } + joined = serving == harness.store.pool.DesiredInstances + } + if !joined { + t.Fatalf("the joining member was delayed behind %d pending bindings: %v", tenants, harness.phases()) + } +} + +// A tenant can lose its last projectable login while its publication is still +// executing at the Gateway. +// +// The stored request is what makes that settleable: there is nothing left in +// the desired state to send, but the occurrence's own body is still there, so +// it is replayed byte for byte and reaches a definite outcome. Abandoning it +// instead - the only option before the body was stored - left the delayed +// original free to commit after the revocation and rebind principals for a +// tenant nobody serves. +func TestALastLoginRemovedWhileAPublicationIsInFlightIsStillSettled(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + // A second login, whose publication is left executing at the Gateway. + tenants.orgs[0].Users = append(tenants.orgs[0].Users, + configstore.TrinoOrgUser{Username: "engineer", PasswordHash: "hash"}) + inFlight := trinoPoolTenantBindingFor(tenants.orgs[0]).Principals + harness.gateway.deferPublish = map[string]bool{"org-a": true} + for tick := 0; tick < 12 && harness.publications.rows["org-a"].PendingIntent == ""; tick++ { + harness.tickTolerant(1) + } + if harness.publications.rows["org-a"].PendingIntent != configstore.TrinoPublicationIntentPrincipals { + t.Fatalf("no publication is in flight: %+v", harness.publications.rows["org-a"]) + } + + // Now the warehouse leaves the projection entirely: there is no desired + // binding left to send under that occurrence. + tenants.orgs = nil + for tick := 0; tick < 24; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + // It was settled by replaying its own stored request, and then revoked. + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, inFlight) { + t.Fatalf("the Gateway binds %v, want the set the occurrence stood for %v", got, inFlight) + } + if !harness.gateway.revoked["org-a"] { + t.Fatalf("the departed tenant was never revoked: %v", harness.gateway.calls) + } + row := harness.publications.rows["org-a"] + if row.PendingIntent != "" || row.PendingPayload != "{}" { + t.Fatalf("publication = %+v, want no request in flight", row) + } + + // And the delayed original, arriving now, changes nothing. + harness.gateway.deliverDeferred() + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, inFlight) { + t.Fatalf("the late duplicate rebound the tenant to %v", got) + } +} + +// Revoked, then re-enabled with a DIFFERENT set, while the revocation is still +// executing at the Gateway. +// +// The revocation is settled from its own stored body first, so the new binding +// and the admission that follows are strictly after it. The late duplicate is a +// replay and cannot take the warehouse back off the air. +func TestAReenabledTenantWithADifferentSetSettlesTheOldRevocationFirst(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + all := tenants.orgs + tenants.orgs = nil + harness.gateway.deferRevoke = map[string]bool{"org-a": true} + for tick := 0; tick < 12 && harness.publications.rows["org-a"].PendingIntent == ""; tick++ { + harness.tickTolerant(1) + } + if harness.publications.rows["org-a"].PendingIntent != configstore.TrinoPublicationIntentRevoke { + t.Fatalf("no revocation is in flight: %+v", harness.publications.rows["org-a"]) + } + + // Back, with a different set of logins. + all[0].Users = []configstore.TrinoOrgUser{{Username: "engineer", PasswordHash: "hash"}} + tenants.orgs = all + wanted := trinoPoolTenantBindingFor(all[0]).Principals + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + harness.gateway.deliverDeferred() + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, wanted) { + t.Fatalf("the Gateway binds %v, want the re-enabled tenant's set %v", got, wanted) + } + if harness.gateway.revoked["org-a"] { + t.Fatalf("the late revocation took the re-enabled tenant off the air: %v", harness.gateway.calls) + } + if harness.gateway.admitted["org-a"] == "" { + t.Fatalf("the re-enabled tenant was never admitted again: %v", harness.gateway.calls) + } + if row := harness.publications.rows["org-a"]; row.State != configstore.TrinoPublicationAdmitted { + t.Fatalf("durable publication = %+v, want an admitted tenant", row) + } +} + +// A duplicate that arrives after the tenant is fully admitted must be inert: +// not a rebinding, not a re-admission, not a change of any kind. +func TestALateDuplicateAfterFinalAdmissionChangesNothing(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + tenants.orgs[0].Users = append(tenants.orgs[0].Users, + configstore.TrinoOrgUser{Username: "engineer", PasswordHash: "hash"}) + harness.gateway.deferPublish = map[string]bool{"org-a": true} + for tick := 0; tick < 12 && harness.publications.rows["org-a"].PendingIntent == ""; tick++ { + harness.tickTolerant(1) + } + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + admitted := harness.gateway.admitted["org-a"] + principals := append([]string(nil), harness.gateway.principals["org-a"]...) + if admitted == "" { + t.Fatalf("the tenant was not admitted at its new binding: %v", harness.gateway.calls) + } + + harness.gateway.deliverDeferred() + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, principals) { + t.Fatalf("the late duplicate rebound the tenant from %v to %v", principals, got) + } + if harness.gateway.admitted["org-a"] != admitted { + t.Fatalf("the late duplicate moved the admission from %q to %q", + admitted, harness.gateway.admitted["org-a"]) + } + if harness.gateway.revoked["org-a"] { + t.Fatal("the late duplicate revoked an admitted tenant") + } +} + +// An identifier a tenant no longer binds must be free for whoever legitimately +// holds it next - and freeing it must not depend on provoking a refusal. +// +// A publication carrying that identifier is in flight when the binding shrinks. +// Settling it byte-identically and then publishing the smaller set under a new +// occurrence releases the identifier through two ordinary successes. Sending the +// smaller set under the OLD occurrence instead reaches the same place only by +// way of a changed-intent refusal, which means the ordinary path depends on an +// error the Gateway raises for a genuine bug. +func TestAnIdentifierDroppedFromABindingCanBeTakenByAnotherTenant(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + dropped := "acme.analyst" + + // A second login is added, and its publication is left executing at the + // Gateway. + tenants.orgs[0].Users = append(tenants.orgs[0].Users, + configstore.TrinoOrgUser{Username: "engineer", PasswordHash: "hash"}) + harness.gateway.deferPublish = map[string]bool{"org-a": true} + for tick := 0; tick < 12 && harness.publications.rows["org-a"].PendingIntent == ""; tick++ { + harness.tickTolerant(1) + } + if harness.publications.rows["org-a"].PendingIntent == "" { + t.Fatalf("no publication is in flight: %+v", harness.publications.rows["org-a"]) + } + + // Then the first login is removed: the desired set no longer has it. + tenants.orgs[0].Users = []configstore.TrinoOrgUser{{Username: "engineer", PasswordHash: "hash"}} + wanted := trinoPoolTenantBindingFor(tenants.orgs[0]).Principals + conflicts := harness.gateway.intentConflicts + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + harness.gateway.deliverDeferred() + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, wanted) { + t.Fatalf("the Gateway binds %v, want the current %v", got, wanted) + } + if owner, bound := harness.gateway.principalOf[dropped]; bound { + t.Fatalf("%s is still owned by %s after being dropped from the binding", dropped, owner) + } + if got := harness.gateway.intentConflicts - conflicts; got != 0 { + t.Fatalf("%d changed-intent refusals were needed to settle the binding; that error is an anomaly, not a protocol", got) + } + // And a different tenant can hold it, which the ownership check would have + // refused while it was still bound. + if _, err := harness.gateway.PublishTenantPrincipals(context.Background(), "pool", "org-b", + trinogateway.PublishPrincipalsRequest{ + Step: trinogateway.Step{OperationID: "tenant.org-b", StepID: "principals.a1"}, + Revision: "binding-b", + Principals: []string{dropped}, + }); err != nil { + t.Fatalf("another tenant cannot take the dropped identifier: %v", err) + } +} + +// A refusal is not an unknown outcome. +// +// The Gateway runs a step in one transaction and every check throws before the +// journal write, so a refused call applied nothing and never will: the request +// is over. Holding the occurrence open for it pins the tenant on a body the +// Gateway has already rejected, and since every other queue skips a tenant with +// a request in flight, the corrected binding an operator produces by removing +// the contested login can never be sent. +// +// The reachable case is a principal that belongs to another tenant, which is +// permanent rather than transient: a revocation leaves the Gateway's principal +// rows in place, so a departed tenant keeps owning its identifiers forever. +func TestADefinitelyRefusedBindingCanBeCorrected(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + // A departed tenant still owns one of this tenant's principals. + contested := "acme.analyst" + harness.gateway.principalOf = map[string]string{contested: "org-old"} + harness.tickTolerant(20) + for tick := 0; tick < 30; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + // The operator fixes the cause: the contested login is removed, so the + // desired binding is publishable. + tenants.orgs[0].Users = nil + wanted := trinoPoolTenantBindingFor(tenants.orgs[0]).Principals + for tick := 0; tick < 60; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, wanted) { + row := harness.publications.rows["org-a"] + t.Fatalf("the Gateway binds %v, want the corrected %v (row: pending=%q payload=%s)", + got, wanted, row.PendingIntent, row.PendingPayload) + } + // The refusal closed the occurrence, and it did NOT checkpoint a binding the + // Gateway never accepted. + if row := harness.publications.rows["org-a"]; row.PendingIntent != "" { + t.Fatalf("publication = %+v, want no request in flight", row) + } +} + +// The same pin also makes a deleted warehouse unrevocable: the revocation pass +// skips a tenant with a request in flight, so its logins stay dispatchable at +// the Gateway - the exact failure that pass exists to prevent. +func TestADefinitelyRefusedTenantCanStillBeRevoked(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.gateway.principalOf = map[string]string{"acme.analyst": "org-old"} + harness.tickTolerant(20) + + // The warehouse is deleted: it leaves the projection entirely. + tenants.orgs = nil + for tick := 0; tick < 60; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + if !harness.gateway.revoked["org-a"] { + t.Fatalf("the departed tenant was never revoked; row: %+v", harness.publications.rows["org-a"]) + } +} + +// Which answers close a tenant's occurrence, stated directly. +// +// The line is "did the Gateway decide?", not "which code was it": a refusal +// applied nothing and can never apply, an unknown outcome may still commit, and +// a lost fence is about this process rather than about the request. +func TestOnlyADefiniteRefusalClosesAnOccurrence(t *testing.T) { + for _, testCase := range []struct { + name string + cause error + definite bool + }{ + {"principal conflict", trinogateway.ErrPrincipalConflict, true}, + {"validation", trinogateway.ErrValidation, true}, + {"changed intent", trinogateway.ErrIntentChanged, true}, + {"unknown pool", trinogateway.ErrNotFound, true}, + {"wrong api mode", trinogateway.ErrAPIMode, true}, + {"protocol disabled", trinogateway.ErrPoolDisabled, true}, + // A 503 from the Gateway's handler and one from anything in front of it + // are indistinguishable here, so the request may yet commit. + {"unavailable", trinogateway.ErrUnavailable, false}, + // Definite for the request, but this process may no longer write: the + // term ends and the next leader settles the occurrence by replaying it. + {"stale epoch", trinogateway.ErrStaleEpoch, false}, + // No verdict at all. + {"transport", errors.New("connection reset by peer"), false}, + } { + t.Run(testCase.name, func(t *testing.T) { + wrapped := fmt.Errorf("publish principals for org-a: %w", testCase.cause) + if got := trinoPoolRefusedDefinitively(wrapped); got != testCase.definite { + t.Fatalf("definite = %v, want %v for %v", got, testCase.definite, testCase.cause) + } + }) + } +} diff --git a/controlplane/trino_pool_progress.go b/controlplane/trino_pool_progress.go new file mode 100644 index 000000000..387c3591b --- /dev/null +++ b/controlplane/trino_pool_progress.go @@ -0,0 +1,589 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// One instance, one step per tick. +// +// Every step is: do the external effect, then record what came back. When the +// response is lost the next tick re-reads instead of re-doing, which is why +// each effect is either idempotent by construction (deterministic Kubernetes +// names) or idempotent by identity (the Gateway's operation/step guard). + +// progressInstances advances the first instance that has work to do and reports +// whether it did anything. Doing one at a time keeps a burst of instances from +// issuing a burst of external effects under one lease. +// +// One instance is NOT allowed to stop the pool. Returning on the first error +// ended the tick before the planner ran, so a member that could not make +// progress - a Gateway decision no retry can change, an object nobody can +// delete - held every repair, drain and replacement for as long as it stayed +// broken. A failing instance is recorded and skipped; the errors are reported +// together, and only a lost fence stops the sweep, because after that nothing +// this process writes can land anyway. This is the isolation the tenant loop +// already applies. +func (o *trinoPoolOperator) progressInstances(ctx context.Context, instances []configstore.TrinoPoolInstance) (bool, error) { + var failures []error + for _, instance := range instances { + phase := trinopool.Phase(instance.Phase) + if phase.Terminal() { + continue + } + progressed, err := o.progressInstance(ctx, instance) + if err != nil { + failures = append(failures, fmt.Errorf("instance %s: %w", instance.InstanceID, err)) + if o.fenced { + return true, errors.Join(failures...) + } + slog.Warn("Trino pool instance step failed; continuing with the rest of the pool.", + "pool", o.config.PublicID, "instance", instance.InstanceID, + "phase", instance.Phase, "error", err) + continue + } + if progressed { + return true, errors.Join(failures...) + } + } + return false, errors.Join(failures...) +} + +func (o *trinoPoolOperator) progressInstance(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + switch trinopool.Phase(instance.Phase) { + case trinopool.PhasePending: + return true, o.createResources(ctx, instance) + case trinopool.PhaseCreating: + return o.registerWhenReady(ctx, instance) + case trinopool.PhasePreparing: + return o.validateCandidate(ctx, instance) + case trinopool.PhaseValidating: + return true, o.admitCandidate(ctx, instance) + case trinopool.PhaseAdmitted: + if progressed, err := o.markServing(ctx, instance); progressed || err != nil { + return progressed, err + } + return o.observeHealth(ctx, instance) + case trinopool.PhaseServing, trinopool.PhaseSuspect: + return o.observeHealth(ctx, instance) + case trinopool.PhaseLost: + return o.completeFailureRetirement(ctx, instance) + case trinopool.PhaseFailedPreparing: + return o.cleanupFailedCandidate(ctx, instance) + case trinopool.PhaseDraining: + return o.sealWhenDrained(ctx, instance) + case trinopool.PhaseSealed: + return true, o.claimRetirement(ctx, instance) + case trinopool.PhaseRetiring: + return o.deleteResources(ctx, instance) + default: + return false, nil + } +} + +// createResources instantiates the instance's OWN blueprint snapshot, not the +// pool's current one: a release that landed after this instance was recorded +// must not change what it runs. +func (o *trinoPoolOperator) createResources(ctx context.Context, instance configstore.TrinoPoolInstance) error { + blueprint, err := trinopool.ParseBlueprint([]byte(instance.BlueprintSnapshot)) + if err != nil { + return fmt.Errorf("instance %s has an unreadable blueprint snapshot: %w", instance.InstanceID, err) + } + objects, err := blueprint.Instantiate(o.identityFor(instance.InstanceID)) + if err != nil { + return fmt.Errorf("instantiate %s: %w", instance.InstanceID, err) + } + inventory, err := o.kube(o.lease.Epoch).Apply(ctx, objects) + if err != nil { + return fmt.Errorf("create resources for %s: %w", instance.InstanceID, err) + } + return o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhasePending, trinopool.PhaseCreating, inventoryUpdates(inventory))) +} + +// registerWhenReady waits for the pods, then registers an unroutable PREPARING +// member. The Gateway backend record is created first and INACTIVE: pooled +// registration reads the endpoint from it, and activating it through the legacy +// route would make the backend eligible with no certificate at all. +func (o *trinoPoolOperator) registerWhenReady(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + inventory := inventoryOf(instance) + observed, err := o.kube(o.lease.Epoch).Observe(ctx, inventory) + if err != nil { + return false, fmt.Errorf("observe %s: %w", instance.InstanceID, err) + } + // A lost registration response is resolved by reading the member back, not + // by building the request again. The request carries the coordinator's + // observed pod and boot identity, and the Gateway hashes the whole request + // under the step identity: once the coordinator has restarted in place, a + // rebuilt request carries a different boot id, reports a changed intent, + // and this instance can never leave CREATING while its member holds a live + // slot. What the Gateway recorded is also the only identity a later receipt + // or loss claim may present, so it is adopted verbatim. + if adopted, err := o.adoptRegisteredMember(ctx, instance, observed); adopted || err != nil { + return adopted, err + } + if !observed.CoordinatorReady || observed.ReadyWorkers == 0 || observed.ReadyWorkers != observed.DesiredWorkers { + // Still converging. Not an error, and not something to time out into a + // failure: the plan's budgets already bound how many instances exist. + return false, nil + } + + // The coordinator's process identity is read BEFORE registration, because + // the Gateway binds (podUid, bootId) at registration and later requires the + // admission receipt to carry the identical pair. Registering the pod UID as + // the boot id and admitting with the coordinator's processId made every + // admission fail POOL_NOT_CERTIFIED. There is exactly one authoritative boot + // identity: the processId, which changes on every JVM start. + bootID, err := o.identity(ctx, instance.EndpointURL) + if err != nil { + slog.Info("Trino pool candidate has no readable process identity yet.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "reason", err) + return false, nil + } + + if err := o.gateway.EnsureInactiveBackend(ctx, trinogateway.Backend{ + Name: o.backendName(instance.InstanceID), + ProxyTo: o.endpointFor(instance.InstanceID), + RoutingGroup: o.config.RoutingGroup, + Active: false, + }); err != nil { + return true, fmt.Errorf("register gateway backend for %s: %w", instance.InstanceID, err) + } + + member, err := o.gateway.RegisterMember(ctx, o.config.RoutingGroup, trinogateway.RegisterMemberRequest{ + Step: o.step(instance.InstanceID, "register"), + InstanceID: instance.InstanceID, + BackendName: o.backendName(instance.InstanceID), + URL: o.endpointFor(instance.InstanceID), + PodUID: observed.CoordinatorPodUID, + BootID: bootID, + ConfigRevision: instance.ReleaseID, + RepairFor: instance.RepairFor, + }) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("register member %s: %w", instance.InstanceID, err)) + } + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseCreating, trinopool.PhasePreparing, map[string]any{ + "coordinator_pod_uid": observed.CoordinatorPodUID, + "coordinator_boot_id": bootID, + // The container instance the admitted process runs in. A termination + // record names a container instance, so without this there is nothing + // to correlate one with, and an unrelated restart from before + // admission reads exactly like the death of this process. + "coordinator_container_id": runningCoordinatorContainer(observed, observed.CoordinatorPodUID), + // The Gateway observes the coordinator's node and coordinator ids + // itself at registration and binds the member to them. Recording + // what it returned is the only way a later loss claim can present + // the identical pair; deriving them again would risk a value the + // Gateway never recorded, and the claim would be refused. + "coordinator_node_id": member.NodeID, + "coordinator_id": member.CoordinatorID, + "gateway_incarnation": member.Incarnation, + "gateway_backend_name": member.BackendName, + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) +} + +// adoptRegisteredMember resolves a registration whose response was lost. +// +// The Gateway is authoritative for its own member, so the identity it recorded +// is adopted rather than re-derived - including a boot identity the coordinator +// has already replaced. That is the correct outcome, not a workaround: the +// candidate then fails its validation against the live process and is replaced +// through the path that exists for exactly that, which also releases the live +// slot the member is holding. +func (o *trinoPoolOperator) adoptRegisteredMember( + ctx context.Context, + instance configstore.TrinoPoolInstance, + observed trinoPoolObservation, +) (bool, error) { + member, err := o.gateway.GetMember(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + if errors.Is(err, trinogateway.ErrNotFound) { + // Nothing was registered under this identity, so registration has + // not happened yet and proceeds normally. + return false, nil + } + return true, fmt.Errorf("read back member %s: %w", instance.InstanceID, err) + } + if member.InstanceID != instance.InstanceID { + // Instance identities are never reused, so this cannot happen; adopting + // another member's incarnation would bind this row to a process it was + // never registered for, so it is refused rather than assumed. + return true, fmt.Errorf("read back member %s returned instance %q", instance.InstanceID, member.InstanceID) + } + slog.Info("Trino pool adopted a member whose registration response was lost.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "phase", member.Phase) + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseCreating, trinopool.PhasePreparing, map[string]any{ + "coordinator_pod_uid": member.PodUID, + "coordinator_boot_id": member.BootID, + // Only when the pod the Gateway recorded is still the one running: + // a container id from a different pod would correlate a later + // termination record with the wrong process. + "coordinator_container_id": runningCoordinatorContainer(observed, member.PodUID), + "coordinator_node_id": member.NodeID, + "coordinator_id": member.CoordinatorID, + "gateway_incarnation": member.Incarnation, + "gateway_backend_name": member.BackendName, + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) +} + +// runningCoordinatorContainer is the container instance currently running in the +// named pod, or "" when that pod is not among the observed ones. Empty means the +// termination evidence below has nothing to correlate against and stays unavailable. +func runningCoordinatorContainer(observed trinoPoolObservation, podUID string) string { + if podUID == "" { + return "" + } + for _, pod := range observed.CoordinatorPods { + if pod.UID == podUID { + return pod.RunningContainerID + } + } + return "" +} + +func (o *trinoPoolOperator) validateCandidate(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + observed, err := o.kube(o.lease.Epoch).Observe(ctx, inventoryOf(instance)) + if err != nil { + return false, fmt.Errorf("observe %s: %w", instance.InstanceID, err) + } + validation, err := o.validate(ctx, instance.EndpointURL, observed, o.expectationFor(instance)) + if err != nil { + // A candidate that is not ready yet stays PREPARING and is probed again. + // It holds a live slot, which the surge budget already accounts for. + slog.Info("Trino pool candidate is not ready yet.", + "pool", o.config.PublicID, "instance", instance.InstanceID, "reason", err) + return false, nil + } + // The registered boot identity is what the Gateway will compare the receipt + // against. If the coordinator restarted since registration, this member's + // incarnation is gone: admitting it is impossible, and waiting for it is + // pointless, so the candidate fails and a fresh instance replaces it. + if instance.CoordinatorBootID != "" && validation.ProcessID != instance.CoordinatorBootID { + slog.Warn("Trino pool candidate restarted before admission; failing it.", + "pool", o.config.PublicID, "instance", instance.InstanceID, + "registered", instance.CoordinatorBootID, "observed", validation.ProcessID) + return true, o.failCandidate(ctx, instance, trinopool.PhasePreparing, + "the coordinator process restarted before admission") + } + receipt, err := marshalValidationReceipt(validation) + if err != nil { + return true, err + } + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhasePreparing, trinopool.PhaseValidating, map[string]any{ + "coordinator_node_id": validation.NodeID, + "coordinator_id": validation.CoordinatorID, + "coordinator_boot_id": validation.ProcessID, + "applied_catalog_revision": validation.AppliedRevision, + "validation_receipt": receipt, + "validated_at": nowUTC(), + })) +} + +// admitCandidate is the single certified-activation call. The Gateway enforces +// the budgets, the certificate freshness and any open publication barrier, and +// independently verifies the live process identity. +func (o *trinoPoolOperator) admitCandidate(ctx context.Context, instance configstore.TrinoPoolInstance) error { + validation, err := unmarshalValidationReceipt(instance.ValidationReceipt) + if err != nil { + return fmt.Errorf("instance %s has an unreadable validation receipt: %w", instance.InstanceID, err) + } + request := trinogateway.AdmitMemberRequest{ + Step: o.step(instance.InstanceID, "admit"), + ExpectedGeneration: instance.GatewayGeneration, + Receipt: trinogateway.ValidationReceipt{ + CertificateHash: validation.CertificateHash, + ConfigRevision: instance.ReleaseID, + AuthRevision: validation.AuthRevision, + PodUID: instance.CoordinatorPodUID, + BootID: validation.ProcessID, + NodeID: validation.NodeID, + CoordinatorID: validation.CoordinatorID, + ReadyWorkers: validation.ReadyWorkers, + Checks: validation.Checks, + }, + } + + // Admission is the one step whose lost response is genuinely ambiguous: the + // member may already be ACTIVE. Recording the intent first means the next + // attempt - possibly a different leader - reads the outcome back instead of + // deciding from nothing. + var member trinogateway.Member + if err := o.runDurableStep(ctx, + configstore.TrinoPoolOperationSpec{ + OperationID: "instance:" + instance.InstanceID, + PoolID: o.config.PoolID, + InstanceID: instance.InstanceID, + Kind: configstore.TrinoPoolOperationReplace, + IntentHash: instance.SpecDigest, + }, + // The step identity is the BUSINESS INTENT - this instance, this + // validated process, this revision - and deliberately NOT the authority + // envelope. The expected generation moves whenever the effect actually + // lands, so hashing the whole request would turn the retry after a lost + // response into a permanent "changed intent" conflict, which is exactly + // the case the record exists to resolve. + "admit", trinoPoolAdmitIntent{ + InstanceID: instance.InstanceID, + CertificateHash: validation.CertificateHash, + ConfigRevision: instance.ReleaseID, + BootID: validation.ProcessID, + }, + func(ctx context.Context) (string, error) { + admitted, err := o.gateway.AdmitMember(ctx, o.config.RoutingGroup, instance.InstanceID, request) + if err != nil { + return "", err + } + member = admitted + return fmt.Sprintf(`{"phase":%q,"generation":%d}`, admitted.Phase, admitted.Generation), nil + }, + ); err != nil { + if errors.Is(err, trinogateway.ErrPublicationBarrier) { + // A member joining while a tenant publication is open must + // acknowledge that publication's target revision, which a member + // registered under a release id cannot. The barrier is the thing + // that has to give way: it can be reopened against the membership + // that includes this member, whereas a candidate refused here would + // wait for a barrier that is itself waiting for capacity. + o.retireOpenBarrierForAdmission(ctx, instance.InstanceID, err) + } + return o.dropAuthority(fmt.Errorf("admit member %s: %w", instance.InstanceID, err)) + } + if member.Phase == "" { + // The step was already recorded as complete by an earlier attempt. Read + // the member back rather than trusting the recorded snapshot: the + // Gateway is authoritative for its own state. + current, err := o.gateway.GetMember(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + return fmt.Errorf("read back admitted member %s: %w", instance.InstanceID, err) + } + member = current + } + return o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseValidating, trinopool.PhaseAdmitted, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) +} + +// markServing records that the Gateway considers the member eligible. Serving +// is the Gateway's judgement, not ours: it is what the minimum-serving floor +// counts. +func (o *trinoPoolOperator) markServing(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + member, err := o.gateway.GetMember(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + return false, fmt.Errorf("read member %s: %w", instance.InstanceID, err) + } + if member.Phase != "ACTIVE" || !member.Eligible { + return false, nil + } + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseAdmitted, trinopool.PhaseServing, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) +} + +// sealWhenDrained asks the Gateway what is still pinned to the member. There is +// no drain deadline: a timer that sealed a member with open transactions would +// be a decision to lose them. +func (o *trinoPoolOperator) sealWhenDrained(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + obligations, err := o.gateway.GetObligations(ctx, o.config.RoutingGroup, instance.InstanceID) + if err != nil { + return false, fmt.Errorf("read obligations for %s: %w", instance.InstanceID, err) + } + if !obligations.Drained || obligations.Outstanding() > 0 { + slog.Debug("Trino pool instance is still draining.", + "pool", o.config.PublicID, "instance", instance.InstanceID, + "outstanding", obligations.Outstanding()) + return false, nil + } + member, err := o.gateway.SealMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "seal"), + // The generation comes from the RECORD, not from the obligations read + // above. The Gateway hashes the whole request under the step identity, + // so a seal whose response was lost can only be resolved by repeating + // the identical request - and the obligations of a member that has + // already been sealed report the generation that seal produced. Sending + // that back would be a changed intent, and this member could never + // finish draining. + ExpectedGeneration: instance.GatewayGeneration, + }) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("seal member %s: %w", instance.InstanceID, err)) + } + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseDraining, trinopool.PhaseSealed, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + })) +} + +// claimRetirement takes the irreversible retirement claim. Nothing is deleted +// before this returns: the claim is the only thing that authorizes it. +func (o *trinoPoolOperator) claimRetirement(ctx context.Context, instance configstore.TrinoPoolInstance) error { + member, err := o.gateway.RetireMember(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "retire"), + ExpectedGeneration: instance.GatewayGeneration, + }) + if err != nil { + return o.dropAuthority(fmt.Errorf("retire member %s: %w", instance.InstanceID, err)) + } + receipt, err := marshalRetirementReceipt(member) + if err != nil { + return err + } + return o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseSealed, trinopool.PhaseRetiring, map[string]any{ + "gateway_state": member.Phase, + "gateway_generation": member.Generation, + "retirement_receipt": receipt, + })) +} + +// deleteResources removes the recorded objects and completes retirement only +// once they are verifiably absent - terminating pods included. +func (o *trinoPoolOperator) deleteResources(ctx context.Context, instance configstore.TrinoPoolInstance) (bool, error) { + inventory := inventoryOf(instance) + kube := o.kube(o.lease.Epoch) + if err := kube.Delete(ctx, inventory); err != nil { + return true, fmt.Errorf("delete resources for %s: %w", instance.InstanceID, err) + } + absent, err := kube.ResourcesAbsent(ctx, inventory) + if err != nil { + return true, fmt.Errorf("verify absence for %s: %w", instance.InstanceID, err) + } + if !absent { + // Deletion is in progress. The instance stays RETIRING and keeps its + // slot until absence is observed, so a terminating pod is never counted + // as freed capacity. + return false, nil + } + if _, err := o.gateway.MemberRetired(ctx, o.config.RoutingGroup, instance.InstanceID, trinogateway.MemberStepRequest{ + Step: o.step(instance.InstanceID, "retired"), + ExpectedGeneration: instance.GatewayGeneration, + ResourcesAbsent: true, + }); err != nil { + return true, o.dropAuthority(fmt.Errorf("report retirement of %s: %w", instance.InstanceID, err)) + } + slog.Info("Trino pool instance retired.", "pool", o.config.PublicID, "instance", instance.InstanceID) + o.closeInstanceOperation(ctx, instance.InstanceID, "retired", "") + return true, o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + trinopool.PhaseRetiring, trinopool.PhaseRetired, nil)) +} + +func inventoryOf(instance configstore.TrinoPoolInstance) trinoPoolInventory { + return trinoPoolInventory{ + Namespace: instanceNamespace(instance), + ConfigMapName: instance.ConfigMapName, + ConfigMapUID: instance.ConfigMapUID, + WorkerConfigMapName: instance.WorkerConfigMapName, + WorkerConfigMapUID: instance.WorkerConfigMapUID, + ServiceName: instance.ServiceName, + ServiceUID: instance.ServiceUID, + CoordinatorDeploymentName: instance.CoordinatorDeploymentName, + CoordinatorDeploymentUID: instance.CoordinatorDeploymentUID, + WorkerDeploymentName: instance.WorkerDeploymentName, + WorkerDeploymentUID: instance.WorkerDeploymentUID, + } +} + +func inventoryUpdates(inventory trinoPoolInventory) map[string]any { + return map[string]any{ + "config_map_name": inventory.ConfigMapName, + "config_map_uid": inventory.ConfigMapUID, + "worker_config_map_name": inventory.WorkerConfigMapName, + "worker_config_map_uid": inventory.WorkerConfigMapUID, + "service_name": inventory.ServiceName, + "service_uid": inventory.ServiceUID, + "coordinator_deployment_name": inventory.CoordinatorDeploymentName, + "coordinator_deployment_uid": inventory.CoordinatorDeploymentUID, + "worker_deployment_name": inventory.WorkerDeploymentName, + "worker_deployment_uid": inventory.WorkerDeploymentUID, + } +} + +// trinoPoolAdmitIntent is what an admission MEANS, separate from the authority +// envelope that carries it. +type trinoPoolAdmitIntent struct { + InstanceID string + CertificateHash string + ConfigRevision string + BootID string +} + +// expectationFor is what this instance must prove before it can be admitted. +// +// The catalog revision comes from the pool's DURABLE publication revision, not +// from a constant: a structurally healthy coordinator sitting at an older +// revision is not certified for the current pool, because it would serve a +// catalog set that does not yet include the newest tenant. Bootstrap is the +// natural zero — before anything is published there is nothing to be behind. +func (o *trinoPoolOperator) expectationFor(instance configstore.TrinoPoolInstance) trinoPoolExpectation { + expectation := trinoPoolExpectation{InternalHTTP: true} + if o.pool != nil { + expectation.CatalogRevision = o.pool.PublicationRevision + } + // What the pool's DURABLE record accepts, not what this process last + // projected: a replica's own memory says what it published, which is + // exactly the thing in question when replicas disagree. + if o.acceptedProjection != nil { + expectation.ProjectionDigest = o.acceptedProjection() + } + // The image comes from the instance's OWN snapshot, so a release that + // landed after this instance was created cannot retroactively change what + // it is required to be running. + if blueprint, err := trinopool.ParseBlueprint([]byte(instance.BlueprintSnapshot)); err == nil { + expectation.Image = blueprint.Image + } + return expectation +} + +// failCandidate records a candidate that can never be admitted. +// +// FAILED_PREPARING is the only terminal state reachable without a Gateway +// retirement receipt, and it is sound exactly because the member never admitted +// work: it was refused before activation. The instance stops occupying a live +// slot, so the planner can replace it instead of blocking behind it forever - +// which is what happened while no failure branch existed at all. +func (o *trinoPoolOperator) failCandidate(ctx context.Context, instance configstore.TrinoPoolInstance, from trinopool.Phase, reason string) error { + o.closeInstanceOperation(ctx, instance.InstanceID, "failed", reason) + return o.dropAuthority(o.store.AdvanceTrinoPoolInstance(ctx, o.lease, instance.InstanceID, + from, trinopool.PhaseFailedPreparing, map[string]any{ + "failure_reason": reason, + "last_error": reason, + })) +} + +// closeInstanceOperation marks an instance's durable operation terminal. +// +// An operation that is never closed leaves `terminal_at` NULL forever: the +// table only grows, and nothing can tell work in flight from work whose +// instance has already reached the end of its life. It is best-effort - the +// instance's own phase is the authoritative record - so a failure here is +// logged rather than propagated. +func (o *trinoPoolOperator) closeInstanceOperation(ctx context.Context, instanceID, phase, lastError string) { + if o.operations == nil { + return + } + if err := o.operations.FinishTrinoPoolOperation(ctx, o.lease, "instance:"+instanceID, phase, lastError); err != nil && + !errors.Is(err, configstore.ErrTrinoPoolConflict) { + slog.Debug("Trino pool operation could not be closed.", + "pool", o.config.PublicID, "instance", instanceID, "error", err) + } +} diff --git a/controlplane/trino_pool_projection_authority.go b/controlplane/trino_pool_projection_authority.go new file mode 100644 index 000000000..584680c30 --- /dev/null +++ b/controlplane/trino_pool_projection_authority.go @@ -0,0 +1,215 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "errors" + "fmt" + "os" + "regexp" + "strings" + "sync/atomic" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/provisioner" +) + +// Who may advance a pooled cell's authorization projection. +// +// The projection is produced by EVERY control plane from its own view, and the +// bundle is served by every replica, so the fence needs two different things: +// +// - an ORDER, so a replica can tell that what it holds has been replaced - +// that is the accepted revision in the config store; and +// - an eligible PRODUCER, because an order alone does not help against binary +// skew. An old control plane that wins the lease publishes its own older +// `policy.rego` and would stamp it with a HIGHER revision: the counter +// orders acceptances, not rule sets. +// +// The producer check is equality against the currently DESIRED image, never an +// ordering of image identities: this process may advance the projection only +// while the image it is running is the one the deployment currently wants. Both +// values already exist and come from the same chart helper: the running image +// is handed to the process at STARTUP in DUCKGRES_TRINO_POOL_PUBLISHER_IMAGE, +// and the desired image is one key on the SAME ConfigMap the pool re-reads +// immediately before every publication. The startup value is used rather than +// the pod's own spec because a pod specification can be edited under a running +// process, while the value the process started with cannot: what this binary +// IS does not change after it starts. During a rollout the two disagree in one +// direction or the other +// and nobody publishes: the pooled bundle pauses, OPA keeps its last-good +// bundle, and no older projection is ever accepted. An intentional rollback +// moves the desired value, which makes the older pods eligible again - the +// wanted semantics, not a regression. +// +// Everything here is pooled-only. A legacy cell installs no fence and its +// projection behaves exactly as before. +const ( + // trinoPoolPublisherImageKey is the ConfigMap key charts render from the + // same image helper the Deployment uses. + trinoPoolPublisherImageKey = "publisher-image" + + // envTrinoPoolPublisherImage carries THIS process's own image, rendered by + // the chart from the same helper as the ConfigMap key above. + envTrinoPoolPublisherImage = "DUCKGRES_TRINO_POOL_PUBLISHER_IMAGE" + + trinoPoolProjectionReadBudget = 10 * time.Second +) + +// trinoPoolPinnedImage matches an image pinned by content digest. A floating +// tag names different bytes at different times, so it cannot establish that +// this binary is the desired publisher; only a digest can. +var trinoPoolPinnedImage = regexp.MustCompile(`@sha256:[0-9a-f]{64}$`) + +// trinoPoolAcceptedProjection reads the accepted projection for the serving +// gate. +// +// The read is NOT cached. A cache - even a two-second one - is a window in +// which a replica keeps handing out a projection the pool has already replaced, +// and the downstream consumer does not repair it: OPA's periodic downloader +// activates each bundle inline as it fetches it, so a poll that receives the +// superseded bundle installs the superseded bundle. That is exactly the +// regression this fence exists to prevent, so the freshness of the answer +// cannot be traded for the round trip that produces it. +type trinoPoolAcceptedProjection struct { + store trinoPoolProjectionReader + poolID string +} + +// trinoPoolProjectionReader is the durable record's read side. +type trinoPoolProjectionReader interface { + GetTrinoPoolProjection(ctx context.Context, poolID string) (configstore.TrinoPoolProjection, error) +} + +// digest reports the accepted projection, and whether it could be determined +// at all. A read failure reports NOT known, so the gate refuses to serve: the +// alternative is serving authorization data nobody can confirm is current. +func (a *trinoPoolAcceptedProjection) digest() (string, bool) { + digest, _, known := a.record(context.Background()) + return digest, known +} + +// record reports the accepted projection and the revision it was accepted at. +// A replica that holds exactly the accepted bytes stamps its Secret with that +// revision, so the revision has to come from the same read as the digest. +func (a *trinoPoolAcceptedProjection) record(ctx context.Context) (string, int64, bool) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolProjectionReadBudget) + defer cancel() + projection, err := a.store.GetTrinoPoolProjection(ctx, a.poolID) + if err != nil { + return "", 0, false + } + return projection.AcceptedDigest, projection.AcceptedRevision, true +} + +// trinoPoolProjectionFence is the provisioner's view of the durable record. +// +// Accept refuses - as ErrTrinoProjectionNotAdvanceable - when this process is +// not the one that may advance the projection: it does not hold the pool's +// authority, or it is not running the desired publisher image. That is the +// state every replica but one is in at any moment, so it is not a reconcile +// failure; the replica keeps building and serving, and publishes nothing the +// record has not accepted. +// +// A read that FAILS is a different thing and is returned as an error: on the +// replica holding the authority, being unable to read the desired publisher or +// this pod is a real fault, not a routine refusal. +type trinoPoolProjectionFence struct { + store *configstore.ConfigStore + publicID string + authority *atomic.Pointer[configstore.TrinoPoolLease] + configSource trinoPoolConfigReader + producer *trinoPoolProducerIdentity + accepted *trinoPoolAcceptedProjection +} + +func (f *trinoPoolProjectionFence) Accept( + ctx context.Context, + build func(orgs []configstore.TrinoEnabledOrg) (string, error), +) (int64, error) { + lease := f.authority.Load() + if lease == nil { + return 0, fmt.Errorf("%w: this control plane does not hold the authority for pool %s", + provisioner.ErrTrinoProjectionNotAdvanceable, f.publicID) + } + // Read the desired publisher AFTER authority is held, from the live object: + // a delayed term that still holds a lease is refused by the database's own + // epoch check below, and a stale desired value cannot be carried in from + // boot. + snapshot, err := f.configSource.Snapshot(ctx) + if err != nil { + return 0, fmt.Errorf("read the desired publisher for pool %s: %w", f.publicID, err) + } + eligible, own, err := f.producer.eligible(snapshot.PublisherImage()) + if err != nil { + return 0, err + } + if !eligible { + return 0, fmt.Errorf("%w: this control plane runs %q, which is not the desired publisher %q for pool %s", + provisioner.ErrTrinoProjectionNotAdvanceable, own, snapshot.PublisherImage(), f.publicID) + } + revision, _, err := f.store.AcceptTrinoPoolProjectionWith(ctx, *lease, build) + if err != nil { + // A refused fenced write means the authority moved while this call was + // in flight. Another process is publishing; this one is simply no + // longer the advancer. + if errors.Is(err, configstore.ErrTrinoPoolConflict) { + return 0, fmt.Errorf("%w: the pool authority moved: %w", provisioner.ErrTrinoProjectionNotAdvanceable, err) + } + return 0, err + } + return revision, nil +} + +func (f *trinoPoolProjectionFence) Accepted(ctx context.Context) (string, int64, bool) { + return f.accepted.record(ctx) +} + +// trinoPoolProducerIdentity answers whether THIS process is the eligible +// publisher for a pooled cell. +// +// ownImage is read ONCE, at startup, from the environment. It is what this +// process IS, and nothing observed later can change that: a pod's spec can be +// edited, and a runtime image ID reports the platform-specific manifest the +// node resolved rather than the multi-platform digest the chart names, so +// neither answers the question the fence asks. +type trinoPoolProducerIdentity struct { + ownImage string +} + +// newTrinoPoolProducerIdentity captures this process's own image at startup. +func newTrinoPoolProducerIdentity() *trinoPoolProducerIdentity { + return &trinoPoolProducerIdentity{ownImage: strings.TrimSpace(os.Getenv(envTrinoPoolPublisherImage))} +} + +// eligible reports whether this process may advance the projection, given the +// desired image from the configuration snapshot it is publishing from. +// +// It fails CLOSED on every uncertainty - no desired value, no startup value, or +// either side naming an image by a floating tag instead of a content digest - +// because the failure mode it exists to prevent is an old binary publishing old +// rules, and "I could not check" is indistinguishable from that. A tag can name +// different bytes at different times, so two equal tags are not evidence that +// two processes run the same code. +func (p *trinoPoolProducerIdentity) eligible(desiredImage string) (bool, string, error) { + desired := strings.TrimSpace(desiredImage) + if desired == "" { + return false, "", fmt.Errorf("the pool configuration carries no %q, so the eligible publisher is unknown", trinoPoolPublisherImageKey) + } + if !trinoPoolPinnedImage.MatchString(desired) { + return false, "", fmt.Errorf("the pool configuration's %q is not pinned to a content digest, so it cannot identify the eligible publisher", trinoPoolPublisherImageKey) + } + own := p.ownImage + if own == "" { + return false, "", fmt.Errorf("this process was started without %s, so it cannot identify its own image", envTrinoPoolPublisherImage) + } + if !trinoPoolPinnedImage.MatchString(own) { + return false, own, fmt.Errorf("%s is not pinned to a content digest, so this process cannot prove which code it runs", envTrinoPoolPublisherImage) + } + if own != desired { + return false, own, nil + } + return true, own, nil +} diff --git a/controlplane/trino_pool_projection_authority_test.go b/controlplane/trino_pool_projection_authority_test.go new file mode 100644 index 000000000..9d094a3a1 --- /dev/null +++ b/controlplane/trino_pool_projection_authority_test.go @@ -0,0 +1,257 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/provisioner/opa" +) + +// recordedProjection is the durable record, with the accepted projection under +// the test's control and every read counted. +type recordedProjection struct { + digest atomic.Pointer[string] + reads atomic.Int64 +} + +func (r *recordedProjection) GetTrinoPoolProjection(context.Context, string) (configstore.TrinoPoolProjection, error) { + r.reads.Add(1) + accepted := "" + if held := r.digest.Load(); held != nil { + accepted = *held + } + return configstore.TrinoPoolProjection{AcceptedDigest: accepted, AcceptedRevision: 1}, nil +} + +func (r *recordedProjection) accept(digest string) { r.digest.Store(&digest) } + +// The serving gate reads the durable record on EVERY request. +// +// It used to cache the answer for two seconds, on the argument that the window +// was no worse than propagation delay. It is worse: OPA's periodic downloader +// activates each bundle inline as it is fetched, so a poll that lands inside +// that window does not merely see a stale answer, it INSTALLS the superseded +// authorization data - after a newer projection has already been accepted and a +// warehouse admitted against it. Priming the reader must not make the next +// request answer from what the previous one saw. +func TestTheServingGateDoesNotServeAProjectionItHasAlreadySeenReplaced(t *testing.T) { + record := &recordedProjection{} + record.accept("projection-1") + gate := &trinoPoolAcceptedProjection{store: record, poolID: "pool-1"} + + // Prime the reader: a coordinator polls this replica while it is current. + if digest, known := gate.digest(); !known || digest != "projection-1" { + t.Fatalf("digest = %q known=%v, want the accepted projection", digest, known) + } + primed := record.reads.Load() + + // The pool accepts a new projection elsewhere, and a tenant is admitted + // against it. + record.accept("projection-2") + + if digest, known := gate.digest(); !known || digest != "projection-2" { + t.Fatalf("digest = %q known=%v, want the replaced projection to be refused immediately", digest, known) + } + if record.reads.Load() <= primed { + t.Fatal("the gate answered from a cached read instead of the durable record") + } +} + +const ( + currentImage = "registry.example.invalid/duckgres@sha256:1111111111111111111111111111111111111111111111111111111111111111" + olderImage = "registry.example.invalid/duckgres@sha256:2222222222222222222222222222222222222222222222222222222222222222" +) + +func producerFor(image string) *trinoPoolProducerIdentity { + return &trinoPoolProducerIdentity{ownImage: image} +} + +// The counterexample the ordering alone does not answer: an OLD control plane +// wins the lease. It would publish its own older policy rules - which live in +// its binary, not in any config - and stamp them with a HIGHER revision, so no +// counter can tell that the authorization data went backwards. Only the process +// the deployment currently wants may advance the projection. +func TestAnOlderBinaryMayNotAdvanceTheProjection(t *testing.T) { + older := producerFor(olderImage) + eligible, own, err := older.eligible(currentImage) + if err != nil { + t.Fatalf("eligibility: %v", err) + } + if eligible { + t.Fatal("a control plane running an older image was allowed to publish authorization data") + } + if own != olderImage { + t.Fatalf("own image = %q, want the image this process was started with", own) + } + + current := producerFor(currentImage) + eligible, _, err = current.eligible(currentImage) + if err != nil || !eligible { + t.Fatalf("the desired publisher was refused: eligible=%v err=%v", eligible, err) + } +} + +// The identity this process publishes under is captured from the environment at +// STARTUP and never re-derived. A pod specification can be edited under a +// running process, so what the API says the pod should run is not evidence +// about the code that is actually executing this check. +func TestProducerIdentityComesFromTheStartupEnvironment(t *testing.T) { + t.Setenv(envTrinoPoolPublisherImage, currentImage) + producer := newTrinoPoolProducerIdentity() + if producer.ownImage != currentImage { + t.Fatalf("own image = %q, want the startup value", producer.ownImage) + } + eligible, _, err := producer.eligible(currentImage) + if err != nil || !eligible { + t.Fatalf("the desired publisher was refused: eligible=%v err=%v", eligible, err) + } + + // The desired image moves. The captured identity does not follow it. + t.Setenv(envTrinoPoolPublisherImage, olderImage) + if eligible, _, err := producer.eligible(olderImage); err != nil || eligible { + t.Fatalf("a running process changed its identity mid-flight: eligible=%v err=%v", eligible, err) + } +} + +// An intentional rollback moves the DESIRED image. The older pods become +// eligible again and the newer ones stop - which is the wanted semantics, not a +// regression: it is equality against what the deployment wants, never an +// ordering of image identities. +func TestARollbackMovesEligibilityWithTheDesiredImage(t *testing.T) { + older, newer := producerFor(olderImage), producerFor(currentImage) + + eligible, _, err := older.eligible(olderImage) + if err != nil || !eligible { + t.Fatalf("after a rollback the desired publisher was refused: eligible=%v err=%v", eligible, err) + } + eligible, _, err = newer.eligible(olderImage) + if err != nil { + t.Fatalf("eligibility: %v", err) + } + if eligible { + t.Fatal("a control plane the deployment no longer wants was allowed to publish") + } +} + +// Every uncertainty fails CLOSED. The failure this exists to prevent is an old +// binary publishing old rules, and "I could not check" is indistinguishable +// from it. +func TestProjectionEligibilityFailsClosed(t *testing.T) { + // A partial Argo sync: the pods are new, the ConfigMap key is not there yet. + if eligible, _, err := producerFor(currentImage).eligible(""); err == nil || eligible { + t.Fatal("publication was permitted with no desired publisher declared") + } + // The process was started without its own image. + anonymous := &trinoPoolProducerIdentity{} + if eligible, _, err := anonymous.eligible(currentImage); err == nil || eligible { + t.Fatal("publication was permitted by a process that cannot identify itself") + } + // A floating tag on either side. Two equal tags are not evidence that two + // processes run the same bytes: a tag can be repointed at any time, which is + // precisely the skew the fence exists to catch. + floating := "registry.example.invalid/duckgres:latest" + if eligible, _, err := producerFor(currentImage).eligible(floating); err == nil || eligible { + t.Fatal("publication was permitted against a floating desired image") + } + if eligible, _, err := producerFor(floating).eligible(floating); err == nil || eligible { + t.Fatal("publication was permitted by a process whose own image is a floating tag") + } + // A digest-shaped value that is not a full sha256 digest is not pinned + // either. + truncated := "registry.example.invalid/duckgres@sha256:1111" + if eligible, _, err := producerFor(truncated).eligible(truncated); err == nil || eligible { + t.Fatal("publication was permitted against a truncated digest") + } +} + +// The serving gate applies to the bundle the handler CAPTURED, and to a 304 as +// much as to a 200: a 304 tells OPA to keep what it has, which preserves +// exactly the stale authorization data the fence exists to retire. +func TestBundleHandlerRefusesAProjectionThatIsNoLongerAccepted(t *testing.T) { + built, err := opa.NewBuilder().BuildBundle(opa.GroupCatalogs{"org_42": {"org_42": true}}, nil) + if err != nil { + t.Fatalf("build bundle: %v", err) + } + bundle := opa.NewBundle(built).WithRevision("projection-1") + store := &opa.BundleStore{} + store.Set(bundle) + + accepted := "projection-1" + handler := opa.NewHandler(store, func(*http.Request) bool { return true }) + handler.AcceptedRevision = func() (string, bool) { return accepted, true } + + server := httptest.NewServer(handler) + defer server.Close() + + response, err := server.Client().Get(server.URL) + if err != nil { + t.Fatalf("get: %v", err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want the accepted projection to be served", response.StatusCode) + } + etag := response.Header.Get("ETag") + + // The projection moves on. This replica still holds the previous one. + accepted = "projection-2" + + response, err = server.Client().Get(server.URL) + if err != nil { + t.Fatalf("get after the projection moved: %v", err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 so OPA keeps its last-good bundle", response.StatusCode) + } + + // And a conditional request is refused too: answering 304 would tell OPA to + // keep the stale bundle it already activated. + request, _ := http.NewRequest(http.MethodGet, server.URL, nil) + request.Header.Set("If-None-Match", etag) + response, err = server.Client().Do(request) + if err != nil { + t.Fatalf("conditional get: %v", err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("conditional status = %d, want 503 rather than a 304 that preserves the stale bundle", response.StatusCode) + } + + // An unreadable record is also a refusal: serving authorization data nobody + // can confirm is current is the thing being prevented. + handler.AcceptedRevision = func() (string, bool) { return "", false } + response, err = server.Client().Get(server.URL) + if err != nil { + t.Fatalf("get with an unreadable record: %v", err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 when the accepted projection cannot be read", response.StatusCode) + } +} + +// A legacy cell installs no gate, and its bundle serving is unchanged. +func TestBundleHandlerIsUnchangedWithoutAFence(t *testing.T) { + built, _ := opa.NewBuilder().BuildBundle(opa.GroupCatalogs{"org_42": {"org_42": true}}, nil) + store := &opa.BundleStore{} + store.Set(opa.NewBundle(built)) + + server := httptest.NewServer(opa.NewHandler(store, func(*http.Request) bool { return true })) + defer server.Close() + + response, err := server.Client().Get(server.URL) + if err != nil { + t.Fatalf("get: %v", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want an unfenced cell to serve exactly as before", response.StatusCode) + } +} diff --git a/controlplane/trino_pool_publication.go b/controlplane/trino_pool_publication.go new file mode 100644 index 000000000..fbbf55bd2 --- /dev/null +++ b/controlplane/trino_pool_publication.go @@ -0,0 +1,1379 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// The tenant publication barrier. +// +// Publishing a tenant's principals tells the Gateway which logins belong to it. +// It does NOT admit the tenant: the Gateway's admission gate dispatches work +// only for a tenant in state ADMITTED, and only a committed publication puts it +// there. Without this driver the gate, once enabled, denies every tenant +// forever - which is why it is here and not a helper waiting for a caller. +// +// The barrier is what makes admission mean something: it commits only when +// EVERY currently active member has acknowledged the tenant's configuration, so +// a tenant is never dispatchable to a coordinator that has not got its catalog, +// its password line or its authorization data yet. +// +// Order per tick, at most one external step, and the order is the priority: +// +// 1. release the live barrier while a member is waiting to join - compute +// first, because an open publication is what refuses the join; +// 2. advance THE one live barrier by one step - open, one receipt, or commit; +// 3. revoke a tenant that has disappeared from the projection (never skip it +// silently: its logins would stay dispatchable); +// 4. publish a changed principal binding; +// 5. open a barrier for the next tenant that needs one. +// +// Finishing the live barrier BEFORE servicing new bindings is what bounds the +// wait. Servicing bindings first meant a fleet with a thousand pending tenants +// spent a thousand passes publishing before it advanced the barrier it already +// had open - and every one of those passes was a pass in which a joining member +// stayed refused. The bound now depends on the member count, not on how many +// tenants happen to be pending. +// +// Three scheduling rules make that safe at fleet scale, and all are +// load-bearing rather than tuning: +// +// - At most ONE barrier is live at a time, and it is driven to completion +// rather than round-robined between steps. Every OPEN publication blocks +// every member admission (the Gateway requires a joining member to +// acknowledge the open publication's target revision, which a member +// registered under a release id can never do), so N concurrent barriers are +// N obstacles to the pool's own compute lifecycle. +// - While a member is waiting to join, no new barrier is opened and the live +// one is released, ONE per pass. Compute wins: a tenant waiting a few more +// seconds is cheaper than a pool that cannot grow back. +// - A tenant with a request in flight is worked ONLY on that request. A lost +// response is not a finished request: it may still be executing at the +// Gateway, and moving to the next desired intent under a new step identity +// would let the older one commit last. +// +// Every decision reads the DURABLE record, never a leader's memory: a restart +// or a leadership move must not republish blindly or assume an admission that +// never committed. + +// trinoPoolTenantStore is the org projection the principal binding is derived +// from. It is the SAME projection that writes the coordinator's password file, +// so the gate can never key on a principal Trino would reject or miss one it +// authenticates. +type trinoPoolTenantStore interface { + ListTrinoEnabledOrgs() ([]configstore.TrinoEnabledOrg, error) +} + +// trinoPoolPublicationStore is the durable publication state. +type trinoPoolPublicationStore interface { + ListTrinoPoolPublications(ctx context.Context, poolID string) ([]configstore.TrinoPoolPublication, error) + RecordTrinoPoolTenantPrincipals(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID, principalRevision string) error + RecordTrinoPoolPublicationOpen(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID, publicationID, targetRevision string) error + RecordTrinoPoolPublicationCommitted(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID, targetRevision, receipt string) error + RecordTrinoPoolTenantRevoked(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID, reason string) error + // ClearTrinoPoolPublicationBarrier is the durable half of abandoning an + // attempt: without it the row keeps naming a finished publication and every + // later pass selects that same dead attempt again. + ClearTrinoPoolPublicationBarrier(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID string) error + // BeginTrinoPoolPublicationAttempt bumps the tenant's occurrence counter, + // which every durable step identity for that tenant carries. + BeginTrinoPoolPublicationAttempt(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID string) (int64, error) + // BeginTrinoPoolPublicationIntent opens an occurrence for a request whose + // outcome will be unknown until the Gateway answers, and records which kind + // of request it stands for. ResolveTrinoPoolPublicationIntent closes it once + // the answer is definite. + BeginTrinoPoolPublicationIntent(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID, kind, payload string) (int64, error) + ResolveTrinoPoolPublicationIntent(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID string) error + // RecordTrinoPoolPublicationFailure / ClearTrinoPoolPublicationFailure are + // the per-tenant durable backoff: the driver takes one tenant at a time, so + // a failing tenant must step aside rather than hold the queue. + RecordTrinoPoolPublicationFailure(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID string, nextAttemptAt time.Time, lastError string) error + ClearTrinoPoolPublicationFailure(ctx context.Context, lease configstore.TrinoPoolLease, poolID, orgID string) error +} + +// trinoPoolAcknowledgement is what one member reports about the configuration +// it is serving. +type trinoPoolAcknowledgement struct { + ProcessID string + AppliedRevision int64 + // ProjectionCurrent reports that this member's authorization and + // authentication data are the ones this control plane is serving. + ProjectionCurrent bool +} + +// trinoPoolBarrierBasis is the configuration ONE attempt is admitting against. +// +// It is captured when the barrier opens and every receipt of that attempt is +// verified against it, so the receipts a commit rests on describe one coherent +// configuration rather than whatever happened to be current when each was +// taken. An attempt whose basis is gone (a leadership move) or no longer +// current (the projection moved) is released and reopened rather than +// completed on mixed evidence. +type trinoPoolBarrierBasis struct { + Projection trinoPoolProjectionRevisions + CatalogRevision int64 +} + +// trinoPoolRevocationReason is fixed, so a revocation reissued under the same +// occurrence carries the identical body the Gateway journaled. +const trinoPoolRevocationReason = "the warehouse is no longer served by this pool" + +// trinoPoolPendingRequest is the stored body of the request an occurrence +// stands for. +// +// It is kept so a reissue is BYTE-IDENTICAL to the original. Sending a +// different body under the same identity would work - the Gateway refuses it +// and that refusal is definite - but it makes a conflict the ordinary success +// path, and it leaves a tenant whose last login disappeared mid-flight with +// nothing to send at all. Storing the request removes both. +// +// Principal identifiers and the revision naming them. Never a credential. +type trinoPoolPendingRequest struct { + Revision string `json:"revision,omitempty"` + Principals []string `json:"principals,omitempty"` + Reason string `json:"reason,omitempty"` +} + +func (r trinoPoolPendingRequest) encode() (string, error) { + encoded, err := json.Marshal(r) + if err != nil { + return "", fmt.Errorf("encode the pending request: %w", err) + } + return string(encoded), nil +} + +func trinoPoolDecodePendingRequest(payload string) (trinoPoolPendingRequest, error) { + var request trinoPoolPendingRequest + if strings.TrimSpace(payload) == "" { + return request, errors.New("the stored request is empty") + } + if err := json.Unmarshal([]byte(payload), &request); err != nil { + return request, fmt.Errorf("decode the stored request: %w", err) + } + return request, nil +} + +// advanceTenantAdmissions drives the barrier for this pool's tenants. +func (o *trinoPoolOperator) advanceTenantAdmissions(ctx context.Context) error { + if !o.config.Pool.TenantAdmission || o.tenants == nil || o.publications == nil { + // The gate is off for this pool, so there is nothing to admit against. + // Publishing a binding anyway would suggest a guarantee the pool is not + // making. + return nil + } + orgs, err := o.tenants.ListTrinoEnabledOrgs() + if err != nil { + return fmt.Errorf("list tenants for pool %s: %w", o.config.PublicID, err) + } + // The admission gate certifies members against the published catalog + // revision, so that number has to be the one the catalog STORE is at - not + // the last one a publication managed to write down. + if err := o.ensureCatalogWatermark(ctx); err != nil { + return err + } + bindings := trinoPoolBindingsFor(orgs, o.config.PoolID) + recorded, err := o.publications.ListTrinoPoolPublications(ctx, o.config.PoolID) + if err != nil { + return fmt.Errorf("read publications for pool %s: %w", o.config.PublicID, err) + } + state := make(map[string]configstore.TrinoPoolPublication, len(recorded)) + for _, publication := range recorded { + state[publication.OrgID] = publication + } + // The instances are read once and reused: the receipt step needs them, and + // whether a candidate is waiting to be admitted decides whether a barrier + // may be open at all. + instances, err := o.store.ListTrinoPoolInstances(ctx, o.config.PoolID) + if err != nil { + return fmt.Errorf("list pool instances: %w", err) + } + + // The live barrier comes first, and freeing a blocked member comes before + // even that. Both are bounded by the MEMBER count; the binding work below is + // bounded by the tenant count, and letting it go first let a fleet's worth + // of pending tenants hold a barrier - and a candidate - open indefinitely. + holder, live := trinoPoolBarrierHolder(recorded) + if live { + return o.serviceLiveBarrier(ctx, bindings, holder, instances) + } + if progressed, err := o.resolvePendingIntent(ctx, bindings, recorded); progressed || err != nil { + return err + } + if progressed, err := o.revokeDepartedTenant(ctx, bindings, recorded); progressed || err != nil { + return err + } + // Publishing a binding and opening the next barrier are two queues, and at + // fleet scale both are long. Taking one of them first whenever it has work + // starves the other: a thousand tenants waiting for a first publication + // would push every admission behind all of them, and a thousand tenants + // waiting for a barrier would do the same to the newest binding. They take + // turns instead, so each drains at half the rate rather than one of them + // not at all. + publish := func() (bool, error) { return o.publishChangedBindings(ctx, bindings, state) } + open := func() (bool, error) { + if trinoPoolAwaitsMemberAdmission(instances) { + // Nothing is open to release, and opening one now would refuse the + // candidate that is waiting. + return false, nil + } + return o.openNextBarrier(ctx, bindings, state) + } + o.tenantTurn++ + queues := [2]func() (bool, error){publish, open} + if o.tenantTurn%2 == 1 { + queues[0], queues[1] = queues[1], queues[0] + } + for _, queue := range queues { + if progressed, err := queue(); progressed || err != nil { + return err + } + } + return nil +} + +// openOccurrence opens a NEW occurrence for a tenant's next request. +// +// Only a tenant whose previous request reached a definite outcome gets here: +// one with a request still in flight is taken by resolvePendingIntent instead, +// which reissues THAT occurrence. The occurrence and what it stands for are +// recorded in ONE write, so a leader that dies between them cannot leave an +// occurrence nobody can attribute. +func (o *trinoPoolOperator) openOccurrence( + ctx context.Context, + orgID, kind string, + request trinoPoolPendingRequest, +) (int64, error) { + payload, err := request.encode() + if err != nil { + return 0, err + } + return o.publications.BeginTrinoPoolPublicationIntent(ctx, o.lease, o.config.PoolID, orgID, kind, payload) +} + +// failTenantStep records the wait a failed request earned, and closes its +// occurrence when - and only when - the Gateway's answer was definite. +// +// A REFUSAL is definite. The Gateway runs a step in one transaction and every +// check throws before the journal write, so a refused call applied nothing and +// never will: that request is over, whatever is on the wire behind it. Holding +// the occurrence open for it pins the tenant on a body the Gateway has already +// rejected - and since every other queue skips a tenant with a request in +// flight, neither a corrected binding nor a revocation could ever be sent. The +// reachable case is a principal owned by another tenant, which does not heal on +// its own: a revocation leaves the Gateway's principal rows in place. +// +// An UNKNOWN outcome keeps the pin, which is what the occurrence exists for. +// Two answers count as unknown: +// +// - no Gateway verdict at all (a transport error), which trinoPoolDecided +// already distinguishes - it is the same question runDurableStep asks; +// - ErrUnavailable, because a 503 from the Gateway's own handler and one from +// anything in front of it are indistinguishable here, so the effect may yet +// commit. +// +// A stale epoch is excluded for a different reason: the refusal is definite, +// but this process has lost the authority to write anything. The term ends and +// the next leader settles the occurrence by replaying it. +// +// Closing an occurrence NEVER moves the checkpoint: the tenant's recorded +// binding must keep describing what the Gateway actually accepted. +func (o *trinoPoolOperator) failTenantStep( + ctx context.Context, + orgID string, + publication configstore.TrinoPoolPublication, + cause error, + what string, + firstAttempt bool, +) error { + // A refusal settles the occurrence only when nothing older can still be + // executing under it, which is true exactly on the FIRST request of an + // occurrence. + // + // On a REISSUE an earlier copy of the same request may still be on its way: + // the Gateway rolls a refused call back before it journals anything, so the + // step identity stays unclaimed and `publishTenantPrincipals` carries no + // revision ordering of its own. Closing here would let the controller + // advance to a newer intent, publish it, checkpoint it - and then have that + // older copy land and overwrite the Gateway's binding with the superseded + // principal set, which duckgres would never correct because it believes it + // already published the newer one. + if firstAttempt && trinoPoolRefusedDefinitively(cause) { + if errors.Is(cause, trinogateway.ErrIntentChanged) { + // With every reissue carrying the stored body this means something + // else recorded this step with different content. + slog.Error("Trino pool tenant occurrence carries content the Gateway did not record for it.", + "pool", o.config.PublicID, "tenant", orgID, + "occurrence", publication.Attempt, "error", cause) + } else { + slog.Warn("Trino pool tenant request was refused; closing its occurrence so the next intent can be sent.", + "pool", o.config.PublicID, "tenant", orgID, "occurrence", publication.Attempt, + "intent", publication.PendingIntent, "error", cause) + } + if err := o.publications.ResolveTrinoPoolPublicationIntent(ctx, o.lease, + o.config.PoolID, orgID); err != nil { + return o.dropAuthority(err) + } + } + delay := trinoPoolRetryDelay(publication.Attempts) + if err := o.publications.RecordTrinoPoolPublicationFailure(ctx, o.lease, + o.config.PoolID, orgID, nowUTC().Add(delay), cause.Error()); err != nil { + return o.dropAuthority(err) + } + return fmt.Errorf("%s: %w", what, cause) +} + +// trinoPoolRefusedDefinitively reports an answer that settles the request: the +// Gateway decided, nothing was applied, and nothing in flight under that +// identity can apply later either. +// +// It reuses the existing decision test rather than an enumeration of codes, so +// a refusal the Gateway adds later is handled the same way it classifies every +// other one. The two exclusions are the answers that are not verdicts about the +// request: see failTenantStep. +func trinoPoolRefusedDefinitively(cause error) bool { + if errors.Is(cause, trinogateway.ErrUnavailable) || errors.Is(cause, trinogateway.ErrStaleEpoch) { + return false + } + // A typed Gateway error IS the verdict, whatever code it carries - the same + // question runDurableStep asks - so a refusal the Gateway adds later needs + // no change here. + if trinoPoolDecided(cause) { + return true + } + // The sentinels a refusal of THESE two calls can carry, for a caller that + // hands back the sentinel without the typed envelope. Anything else stays + // unknown, which keeps the occurrence pinned: the conservative direction. + for _, refusal := range []error{ + trinogateway.ErrPrincipalConflict, + trinogateway.ErrValidation, + trinogateway.ErrIntentChanged, + trinogateway.ErrNotFound, + trinogateway.ErrAPIMode, + trinogateway.ErrPoolDisabled, + trinogateway.ErrIdentityConflict, + trinogateway.ErrTenantNotAdmitted, + } { + if errors.Is(cause, refusal) { + return true + } + } + return false +} + +// trinoPoolAwaitsMemberAdmission reports that a candidate is sitting at the +// Gateway's admission call. +// +// A candidate stays VALIDATING until its admission is accepted, so this is +// exactly "a member is blocked joining" - read from the durable instance rows +// rather than remembered from the last refusal, which a leadership move would +// lose and a durable admission backoff would delay by up to its whole wait. +func trinoPoolAwaitsMemberAdmission(instances []configstore.TrinoPoolInstance) bool { + for _, instance := range instances { + if trinopool.Phase(instance.Phase) == trinopool.PhaseValidating { + return true + } + } + return false +} + +// resolvePendingIntent finishes the ONE request a tenant already has in flight, +// before any tenant's desired intent is allowed to move. +// +// A lost response is not a finished request: it may still be executing at the +// Gateway and commit whenever it gets there. Two requests from the same +// controller under different step identities are unordered there - the row lock +// serializes arrival, not desire - so issuing the next intent while the +// previous one is unresolved lets the older one commit last: a binding nobody +// wants, or a revocation of a tenant that has since been re-enabled, while this +// control plane has checkpointed the newer intent and will never issue it +// again. +// +// Reissuing the SAME occurrence is what settles it. Either the Gateway has the +// step journaled - a definite answer, and the still-delayed duplicate can only +// replay it - or this call records it, and the delayed duplicate arrives under +// a journaled identity and applies nothing. +func (o *trinoPoolOperator) resolvePendingIntent( + ctx context.Context, + bindings []trinoPoolTenantBinding, + recorded []configstore.TrinoPoolPublication, +) (bool, error) { + now := nowUTC() + for _, publication := range recorded { + if publication.PendingIntent == "" { + continue + } + if publication.NextAttemptAt != nil && now.Before(*publication.NextAttemptAt) { + // Serving out the wait its last failure earned. Another tenant may + // proceed meanwhile; this one resumes when the wait elapses. + continue + } + request, err := trinoPoolDecodePendingRequest(publication.PendingPayload) + if err != nil { + // The stored request is what makes the reissue possible, so an + // unreadable one cannot be settled by replay. Closing the occurrence + // is the only move left; it is also a bug, so it is loud. + slog.Error("Trino pool cannot replay a tenant's request and is closing its occurrence.", + "pool", o.config.PublicID, "tenant", publication.OrgID, + "occurrence", publication.Attempt, "intent", publication.PendingIntent, "error", err) + return true, o.dropAuthority(o.publications.ResolveTrinoPoolPublicationIntent(ctx, o.lease, + o.config.PoolID, publication.OrgID)) + } + if publication.PendingIntent == configstore.TrinoPublicationIntentRevoke { + return true, o.reissueRevoke(ctx, publication, request) + } + // The STORED set, not the desired one. A tenant whose last login has + // since gone away still has exactly this to replay, which is what makes + // the occurrence settleable rather than abandoned - and an abandoned + // occurrence is what let a delayed publication rebind principals nobody + // wanted afterwards. + return true, o.reissuePublication(ctx, publication, request) + } + return false, nil +} + +// reissueRevoke settles a revocation whose outcome is unknown. It runs even +// when the tenant has come back: the revocation has to reach a definite outcome +// before the tenant can be published and admitted again, or it could commit +// afterwards and take a serving warehouse off the air with nothing left in the +// desired state to correct it. +func (o *trinoPoolOperator) reissueRevoke( + ctx context.Context, + publication configstore.TrinoPoolPublication, + request trinoPoolPendingRequest, +) error { + reason := request.Reason + if _, err := o.gateway.RevokeTenant(ctx, o.config.RoutingGroup, publication.OrgID, trinogateway.RevokeTenantRequest{ + Step: o.step("tenant."+publication.OrgID, trinoPoolStepID("revoke", trinoPoolOccurrence(publication.Attempt))), + Reason: reason, + }); err != nil { + return o.failTenantStep(ctx, publication.OrgID, publication, err, + fmt.Sprintf("settle the revocation of %s", publication.OrgID), false) + } + slog.Info("Trino pool tenant revocation settled.", + "pool", o.config.PublicID, "tenant", publication.OrgID, "occurrence", publication.Attempt) + o.forgetBarrierBasis(publication.PublicationID) + return o.dropAuthority(o.publications.RecordTrinoPoolTenantRevoked(ctx, o.lease, + o.config.PoolID, publication.OrgID, reason)) +} + +// reissuePublication settles a publication whose outcome is unknown by sending +// the STORED request again, byte for byte. +// +// An identical body under the same identity is an ordinary replay: if the +// earlier call committed, the Gateway returns its recorded result, and if it did +// not, this one records the identity so the delayed original applies nothing +// when it arrives. Neither outcome depends on a refusal, and a tenant whose +// desired binding has changed - or vanished - since is settled just the same. +// Its new binding, if any, goes out afterwards under a new occurrence. +func (o *trinoPoolOperator) reissuePublication( + ctx context.Context, + publication configstore.TrinoPoolPublication, + request trinoPoolPendingRequest, +) error { + admission, err := o.gateway.PublishTenantPrincipals(ctx, o.config.RoutingGroup, publication.OrgID, + trinogateway.PublishPrincipalsRequest{ + Step: o.step("tenant."+publication.OrgID, trinoPoolStepID("principals", trinoPoolOccurrence(publication.Attempt))), + Revision: request.Revision, + Principals: request.Principals, + }) + if err != nil { + if o.fenced { + return err + } + return o.failTenantStep(ctx, publication.OrgID, publication, err, + fmt.Sprintf("settle the publication for %s", publication.OrgID), false) + } + slog.Info("Trino pool tenant binding settled.", + "pool", o.config.PublicID, "tenant", publication.OrgID, + "principals", len(request.Principals), "occurrence", publication.Attempt, "state", admission.State) + if err := o.publications.RecordTrinoPoolTenantPrincipals(ctx, o.lease, + o.config.PoolID, publication.OrgID, request.Revision); err != nil { + return o.dropAuthority(err) + } + return o.dropAuthority(o.publications.ClearTrinoPoolPublicationFailure(ctx, o.lease, + o.config.PoolID, publication.OrgID)) +} + +// revokeDepartedTenant withdraws the admission of a tenant that is no longer in +// the projection. +// +// Skipping it would leave the tenant's principals dispatchable indefinitely: the +// Gateway replaces a principal set only when it is published, so a warehouse +// that was disabled or deleted keeps its logins admitted until somebody says +// otherwise. One per tick, because each is an external mutation. +func (o *trinoPoolOperator) revokeDepartedTenant( + ctx context.Context, + bindings []trinoPoolTenantBinding, + recorded []configstore.TrinoPoolPublication, +) (bool, error) { + present := make(map[string]bool, len(bindings)) + for _, binding := range bindings { + // A tenant whose last projectable login was removed is DEPARTED, not + // "nothing to do": its principals stay dispatchable until somebody + // publishes over them, and an empty set is not publishable. Skipping it + // left a warehouse admitted under logins that no longer exist. + if len(binding.Principals) == 0 { + continue + } + present[binding.Tenant] = true + } + now := nowUTC() + for _, publication := range recorded { + if present[publication.OrgID] || publication.State == configstore.TrinoPublicationRevoked { + continue + } + if publication.PendingIntent != "" { + // Its open occurrence is settled first, by the resolver. + continue + } + if publication.NextAttemptAt != nil && now.Before(*publication.NextAttemptAt) { + continue + } + reason := trinoPoolRevocationReason + attempt, err := o.openOccurrence(ctx, publication.OrgID, configstore.TrinoPublicationIntentRevoke, + trinoPoolPendingRequest{Reason: reason}) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("start a revocation for %s: %w", publication.OrgID, err)) + } + if _, err := o.gateway.RevokeTenant(ctx, o.config.RoutingGroup, publication.OrgID, trinogateway.RevokeTenantRequest{ + // The occurrence is what makes a SECOND revocation - after the + // tenant was re-enabled and admitted again - a new operation rather + // than a replay that returns the first revocation's outcome and + // leaves the tenant admitted. It is PINNED until the Gateway answers + // definitively, so a revocation still executing there cannot commit + // after the tenant has been re-enabled and re-admitted. + Step: o.step("tenant."+publication.OrgID, trinoPoolStepID("revoke", trinoPoolOccurrence(attempt))), + Reason: reason, + }); err != nil { + return true, o.failTenantStep(ctx, publication.OrgID, publication, err, + fmt.Sprintf("revoke tenant %s", publication.OrgID), true) + } + slog.Info("Trino pool tenant admission revoked.", + "pool", o.config.PublicID, "tenant", publication.OrgID, "occurrence", attempt) + o.forgetBarrierBasis(publication.PublicationID) + return true, o.dropAuthority(o.publications.RecordTrinoPoolTenantRevoked(ctx, o.lease, + o.config.PoolID, publication.OrgID, reason)) + } + return false, nil +} + +// publishChangedBindings publishes one tenant's principal set when it differs +// from the DURABLE record of what was last published. +// +// Every publication is a NEW durable occurrence, taken before the call. The +// Gateway resolves a step identity it has already recorded by returning that +// step's result and applying NOTHING, and a tenant's revision is a digest of +// its principal set - so a login that is added and then removed again produces +// a set, and therefore a body, byte-identical to an earlier one. Under a +// constant step identity that publication is a replay: the removed login stays +// bound to the tenant at the Gateway, and the next tenant to be given that +// principal is refused for a conflict it cannot see. The same applies to +// re-publishing after a revocation. +func (o *trinoPoolOperator) publishChangedBindings( + ctx context.Context, + bindings []trinoPoolTenantBinding, + state map[string]configstore.TrinoPoolPublication, +) (bool, error) { + now := nowUTC() + eligible := make([]trinoPoolTenantBinding, 0, len(bindings)) + for _, binding := range bindings { + if len(binding.Principals) == 0 { + // A tenant with no projectable login has nothing to publish: the + // Gateway rejects an empty set, and inventing one would bind a + // principal that cannot authenticate. Its admission is withdrawn by + // the revocation pass instead. + continue + } + publication := state[binding.Tenant] + if publication.PrincipalRevision == binding.Revision && + publication.State != configstore.TrinoPublicationRevoked && + publication.PendingIntent == "" { + continue + } + if publication.PendingIntent != "" { + // Its open occurrence is settled first, by the resolver. Moving to + // the next intent here is exactly what let an older request commit + // last. + continue + } + if publication.NextAttemptAt != nil && now.Before(*publication.NextAttemptAt) { + continue + } + eligible = append(eligible, binding) + } + if len(eligible) == 0 { + return false, nil + } + // Rotate, for the same reason the barrier does: a tenant whose publication + // keeps failing must not hold the front of the queue forever. + o.bindingCursor++ + binding := eligible[int(o.bindingCursor%uint64(len(eligible)))] + publication := state[binding.Tenant] + // The request is stored with the occurrence, so every later attempt at it - + // by this leader or the next - sends these exact bytes. + request := trinoPoolPendingRequest{Revision: binding.Revision, Principals: binding.Principals} + attempt, err := o.openOccurrence(ctx, binding.Tenant, configstore.TrinoPublicationIntentPrincipals, request) + if err != nil { + return true, o.dropAuthority(fmt.Errorf("start a publication for %s: %w", binding.Tenant, err)) + } + // The step identity is the OCCURRENCE, not the body. A publication whose + // response was lost may still be executing at the Gateway; reissuing the + // same identity is what makes that late arrival a journal replay instead of + // a second effect that overwrites the binding published since. + admission, err := o.gateway.PublishTenantPrincipals(ctx, o.config.RoutingGroup, binding.Tenant, + trinogateway.PublishPrincipalsRequest{ + Step: o.step("tenant."+binding.Tenant, trinoPoolStepID("principals", trinoPoolOccurrence(attempt))), + Revision: request.Revision, + Principals: request.Principals, + }) + if err != nil { + if o.fenced { + return true, err + } + return true, o.failTenantStep(ctx, binding.Tenant, publication, err, + fmt.Sprintf("publish principals for %s", binding.Tenant), true) + } + slog.Info("Trino pool tenant binding published.", + "pool", o.config.PublicID, "tenant", binding.Tenant, + "principals", len(binding.Principals), "occurrence", attempt, "state", admission.State) + if err := o.publications.RecordTrinoPoolTenantPrincipals(ctx, o.lease, + o.config.PoolID, binding.Tenant, binding.Revision); err != nil { + return true, o.dropAuthority(err) + } + return true, o.dropAuthority(o.publications.ClearTrinoPoolPublicationFailure(ctx, o.lease, + o.config.PoolID, binding.Tenant)) +} + +// advanceOneBarrier moves THE live barrier one step, or opens one when there is +// none and nothing is waiting on the pool's compute. +// +// Three properties matter at fleet scale, where a pool serves thousands of +// warehouses: +// +// - A tenant's target is ITS OWN, not a snapshot of the whole fleet's +// configuration. An admitted tenant is finished until its own principals +// change; provisioning a NEW warehouse must not re-admit every existing +// one, which at one external step per five-second tick would have delayed +// that new tenant by hours. +// - ONE barrier is live at a time and is driven to completion. Rotating +// between steps spread each tenant's attempt over the whole fleet's +// rotation - long enough for ordinary membership churn to invalidate it +// before it could commit - and left one open publication per eligible +// tenant, every one of which blocks a member from joining. +// - Fairness comes from the durable backoff, not from rotating mid-attempt: +// a tenant whose step failed releases its barrier and serves its wait while +// the next tenant runs. +func (o *trinoPoolOperator) serviceLiveBarrier( + ctx context.Context, + bindings []trinoPoolTenantBinding, + holder configstore.TrinoPoolPublication, + instances []configstore.TrinoPoolInstance, +) error { + now := nowUTC() + if trinoPoolAwaitsMemberAdmission(instances) { + // Compute wins. Every OPEN publication refuses the joining member, and + // below the serving floor that is not a delay but a pool that cannot + // grow back. One release per pass keeps the work bounded however many + // barriers an older version left behind. + return o.releaseBarrier(ctx, holder, "a pool member is waiting to join") + } + binding, known := trinoPoolBindingFor(bindings, holder.OrgID) + switch { + case !known || len(binding.Principals) == 0 || holder.PrincipalRevision != binding.Revision: + return o.releaseBarrier(ctx, holder, "the tenant's binding changed during the attempt") + case holder.PendingIntent != "": + // A request for this tenant is in flight, so what it will be admitted + // FOR is not settled yet. + return o.releaseBarrier(ctx, holder, "the tenant has a request in flight") + case o.tenantIsCurrent(binding, holder): + return o.releaseBarrier(ctx, holder, "the tenant is already admitted at this intent") + case holder.NextAttemptAt != nil && now.Before(*holder.NextAttemptAt): + return o.releaseBarrier(ctx, holder, "the tenant is serving out the wait its last failure earned") + } + err := o.stepLiveBarrier(ctx, binding, holder, instances) + if err == nil { + return o.dropAuthority(o.publications.ClearTrinoPoolPublicationFailure(ctx, o.lease, + o.config.PoolID, binding.Tenant)) + } + if o.fenced { + // Authority loss is not this tenant's problem and must not be recorded + // as one. + return err + } + // Record the wait this tenant earned, then report the failure. The next + // pass releases its barrier and moves on to somebody else. + delay := trinoPoolRetryDelay(holder.Attempts) + if recordErr := o.publications.RecordTrinoPoolPublicationFailure(ctx, o.lease, + o.config.PoolID, binding.Tenant, now.Add(delay), err.Error()); recordErr != nil { + return o.dropAuthority(recordErr) + } + return err +} + +// openNextBarrier opens a barrier for the next tenant that needs one. There is +// no live barrier when this runs, so at most one is ever open. +func (o *trinoPoolOperator) openNextBarrier( + ctx context.Context, + bindings []trinoPoolTenantBinding, + state map[string]configstore.TrinoPoolPublication, +) (bool, error) { + now := nowUTC() + if o.expectedProjection().Policy == "" { + // Nothing has been published yet, so there is no configuration for a + // member to acknowledge. Opening a barrier against an unknown + // projection would admit a tenant against nothing. + return false, nil + } + eligible := make([]trinoPoolTenantBinding, 0, len(bindings)) + for _, binding := range bindings { + publication := state[binding.Tenant] + if len(binding.Principals) == 0 || publication.PrincipalRevision != binding.Revision { + // The binding has to be published before the barrier can mean + // anything: the admission it opens is for that principal set. + continue + } + if publication.PendingIntent != "" { + // A request for this tenant is still in flight. Admitting it now + // would be admitting it for a binding that is not settled. + continue + } + if o.tenantIsCurrent(binding, publication) { + continue + } + if publication.NextAttemptAt != nil && now.Before(*publication.NextAttemptAt) { + // This tenant is serving out the wait its last failure earned. + continue + } + eligible = append(eligible, binding) + } + if len(eligible) == 0 { + return false, nil + } + // Rotate the starting point so no tenant owns the front of the queue. + o.barrierCursor++ + binding := eligible[int(o.barrierCursor%uint64(len(eligible)))] + return true, o.openBarrier(ctx, binding) +} + +// trinoPoolBarrierHolder returns the tenant whose durable row names a live +// barrier. A row names one exactly while its attempt is in flight: the open +// records it, and the commit, the revocation and the release all clear it. +func trinoPoolBarrierHolder(recorded []configstore.TrinoPoolPublication) (configstore.TrinoPoolPublication, bool) { + for _, publication := range recorded { + if publication.PublicationID != "" && publication.State != configstore.TrinoPublicationRevoked { + return publication, true + } + } + return configstore.TrinoPoolPublication{}, false +} + +func trinoPoolBindingFor(bindings []trinoPoolTenantBinding, tenant string) (trinoPoolTenantBinding, bool) { + for _, binding := range bindings { + if binding.Tenant == tenant { + return binding, true + } + } + return trinoPoolTenantBinding{}, false +} + +// releaseBarrier retires the live attempt, at the Gateway AND in the durable +// record, so the next pass is free to do something else. +// +// Both halves are required. Abandoning without clearing leaves the row naming a +// finished publication, which every later pass selects again - the loop that +// kept re-abandoning one tenant's dead barrier while the barrier actually +// blocking a member's admission stayed open. Clearing without abandoning leaves +// an OPEN publication at the Gateway that nothing will ever close. +func (o *trinoPoolOperator) releaseBarrier( + ctx context.Context, + publication configstore.TrinoPoolPublication, + reason string, +) error { + slog.Info("Trino pool publication attempt released.", + "pool", o.config.PublicID, "tenant", publication.OrgID, + "publication", publication.PublicationID, "reason", reason) + _, err := o.gateway.AbandonPublication(ctx, o.config.RoutingGroup, publication.PublicationID, + o.step("publication."+publication.OrgID, trinoPoolStepID("abandon", publication.PublicationID))) + switch { + case err == nil, trinogateway.IsNotFound(err): + // Abandoned, or the Gateway never had it: either way it is finished. + case isTrinoPoolCommittedPublication(err): + // An opened admission gate is never retracted. The tenant IS admitted; + // the checkpoint simply never landed, so it is read back and recorded + // rather than lost. + current, readErr := o.gateway.GetPublication(ctx, o.config.RoutingGroup, publication.PublicationID) + if readErr != nil { + return o.dropAuthority(fmt.Errorf("read back publication %s: %w", publication.PublicationID, readErr)) + } + binding := trinoPoolTenantBinding{Tenant: publication.OrgID} + return o.recordAdmitted(ctx, binding, current.TargetRevision, current) + default: + return o.dropAuthority(fmt.Errorf("abandon publication %s: %w", publication.PublicationID, err)) + } + o.forgetBarrierBasis(publication.PublicationID) + return o.dropAuthority(o.publications.ClearTrinoPoolPublicationBarrier(ctx, o.lease, + o.config.PoolID, publication.OrgID)) +} + +// isTrinoPoolCommittedPublication reports the Gateway refusing to abandon a +// publication because it has already admitted the tenant. +func isTrinoPoolCommittedPublication(err error) bool { + return errors.Is(err, trinogateway.ErrIrreversible) +} + +// openBarrier starts a NEW attempt for this tenant. +// +// The occurrence counter is durable and monotone, and every step identity +// carries the publication it belongs to, so the new attempt shares nothing with +// any it replaces: a new publication, a new open step, new receipts and a new +// commit. Reusing an identity would resolve the previous attempt's recorded +// outcome and leave this intent unapplied. +func (o *trinoPoolOperator) openBarrier(ctx context.Context, binding trinoPoolTenantBinding) error { + poolState, err := o.gateway.GetPool(ctx, o.config.RoutingGroup) + if err != nil { + return fmt.Errorf("read pool state for %s: %w", o.config.PublicID, err) + } + if poolState.ServingMembers < int64(o.config.Spec.MinServing) { + // The Gateway refuses a publication below the serving floor. Opening one + // to be told so is noise; the next tick tries again. + slog.Debug("Trino pool publication is waiting for the serving floor.", + "pool", o.config.PublicID, "tenant", binding.Tenant, "serving", poolState.ServingMembers) + return nil + } + attempt, err := o.publications.BeginTrinoPoolPublicationAttempt(ctx, o.lease, o.config.PoolID, binding.Tenant) + if err != nil { + return o.dropAuthority(fmt.Errorf("start a publication attempt for %s: %w", binding.Tenant, err)) + } + target := trinoPoolTargetRevision(binding, attempt) + publicationID := trinoPoolPublicationID(binding.Tenant, target) + // The identity is recorded BEFORE the Gateway call, so a lost response is + // resolved by reading that publication back instead of opening a second + // barrier - which the Gateway refuses anyway, leaving the first one open + // forever. + if err := o.publications.RecordTrinoPoolPublicationOpen(ctx, o.lease, + o.config.PoolID, binding.Tenant, publicationID, target); err != nil { + return o.dropAuthority(fmt.Errorf("record publication intent for %s: %w", binding.Tenant, err)) + } + o.rememberBarrierBasis(publicationID) + opened, err := o.issueOpenPublication(ctx, binding, target, publicationID, poolState.MembershipGeneration) + if err != nil { + return err + } + slog.Info("Trino pool publication opened.", + "pool", o.config.PublicID, "tenant", binding.Tenant, "publication", publicationID, + "target", target, "attempt", attempt, "required", len(opened.RequiredMembers)) + return nil +} + +func (o *trinoPoolOperator) issueOpenPublication( + ctx context.Context, + binding trinoPoolTenantBinding, + target, publicationID string, + membershipGeneration int64, +) (trinogateway.Publication, error) { + opened, err := o.gateway.OpenPublication(ctx, o.config.RoutingGroup, trinogateway.OpenPublicationRequest{ + Step: o.step("publication."+binding.Tenant, trinoPoolStepID("open", publicationID)), + PublicationID: publicationID, + Tenant: binding.Tenant, + TargetRevision: target, + ExpectedMembershipGeneration: membershipGeneration, + PayloadHash: trinoPoolPublicationPlanHash(binding, target), + }) + if err != nil { + return trinogateway.Publication{}, o.dropAuthority(fmt.Errorf("open publication for %s: %w", binding.Tenant, err)) + } + return opened, nil +} + +// stepLiveBarrier performs the ONE next step of the live attempt. +func (o *trinoPoolOperator) stepLiveBarrier( + ctx context.Context, + binding trinoPoolTenantBinding, + publication configstore.TrinoPoolPublication, + instances []configstore.TrinoPoolInstance, +) error { + poolState, err := o.gateway.GetPool(ctx, o.config.RoutingGroup) + if err != nil { + return fmt.Errorf("read pool state for %s: %w", o.config.PublicID, err) + } + if poolState.ServingMembers < int64(o.config.Spec.MinServing) { + slog.Debug("Trino pool publication is waiting for the serving floor.", + "pool", o.config.PublicID, "tenant", binding.Tenant, "serving", poolState.ServingMembers) + return nil + } + + current, err := o.gateway.GetPublication(ctx, o.config.RoutingGroup, publication.PublicationID) + if err != nil { + if !trinogateway.IsNotFound(err) { + return fmt.Errorf("read publication %s: %w", publication.PublicationID, err) + } + // Recorded but absent on the Gateway: the open never landed. Re-issuing + // it under the SAME identity is the resolution, not a second barrier. + current, err = o.issueOpenPublication(ctx, binding, publication.TargetRevision, + publication.PublicationID, poolState.MembershipGeneration) + if err != nil { + return err + } + o.rememberBarrierBasis(publication.PublicationID) + return nil + } + + switch current.Phase { + case "ADMITTED": + return o.recordAdmitted(ctx, binding, publication.TargetRevision, current) + case "ABANDONED": + // Finished business, not a fault to escalate: membership changes during + // a deployment, and abandoning is how the protocol lets an attempt that + // can no longer commit get out of the way. Clearing the durable pointer + // is what lets the next pass open a new attempt, under a new occurrence. + o.forgetBarrierBasis(publication.PublicationID) + return o.dropAuthority(o.publications.ClearTrinoPoolPublicationBarrier(ctx, o.lease, + o.config.PoolID, binding.Tenant)) + } + if current.MembershipGeneration != poolState.MembershipGeneration { + // The membership this barrier was opened against has moved on: a member + // joined, failed or was replaced. Its commit can never satisfy the + // Gateway's generation check. + return o.releaseBarrier(ctx, publication, "the pool membership changed during the attempt") + } + basis, ok := o.barrierBasisFor(publication.PublicationID) + if !ok { + // This process did not open this attempt (a leadership move), so what + // its existing receipts attest to is unknown. Completing it would rest a + // commit on evidence nobody can describe. + return o.releaseBarrier(ctx, publication, "the configuration this attempt was opened against is unknown") + } + if basis.Projection != o.expectedProjection() { + // Receipts already recorded attest to the projection this attempt was + // opened against. Finishing it against a newer one would commit on + // mixed evidence; the tenant gets a fresh attempt on the current + // projection instead, and an already-admitted tenant is untouched + // because it has no attempt in flight. + return o.releaseBarrier(ctx, publication, "the accepted projection changed during the attempt") + } + if len(current.MissingMembers) > 0 { + return o.recordOneReceipt(ctx, binding, current, basis, instances) + } + return o.commitBarrier(ctx, binding, current, publication.TargetRevision, poolState.MembershipGeneration) +} + +// retireOpenBarrierForAdmission releases the live tenant publication that is +// standing in the way of a member's admission. +// +// The Gateway requires a member joining during an open publication to +// acknowledge that publication's target revision. A member registers under its +// RELEASE id, so it can never satisfy a tenant barrier's target, and the two +// would wait for each other: the member for the barrier to close, the barrier +// for a membership that includes the member. +// +// The scheduler already suppresses new barriers and releases the live one while +// a candidate is waiting, so this is the belt on that brace: it runs when an +// admission has actually been refused, releases exactly ONE attempt, and +// records that release durably so the next refusal reaches the next one instead +// of re-abandoning the same finished publication. +func (o *trinoPoolOperator) retireOpenBarrierForAdmission(ctx context.Context, instanceID string, cause error) { + if o.publications == nil { + return + } + recorded, err := o.publications.ListTrinoPoolPublications(ctx, o.config.PoolID) + if err != nil { + slog.Warn("Trino pool could not read publications while admitting a member.", + "pool", o.config.PublicID, "instance", instanceID, "error", err) + return + } + holder, live := trinoPoolBarrierHolder(recorded) + if !live { + return + } + slog.Info("Trino pool is releasing a tenant publication so a member can be admitted.", + "pool", o.config.PublicID, "tenant", holder.OrgID, + "publication", holder.PublicationID, "instance", instanceID, "reason", cause) + if err := o.releaseBarrier(ctx, holder, "a pool member is waiting to join"); err != nil { + slog.Warn("Trino pool could not release the open publication.", + "pool", o.config.PublicID, "tenant", holder.OrgID, "error", err) + } +} + +// recordOneReceipt acknowledges ONE member, after verifying against that member +// itself that it is serving the configuration the barrier requires. +// +// The verification is the point of the receipt. Asserting acknowledgement from +// what this controller published - rather than from what the member reports +// having loaded - would commit a barrier while a coordinator still lacks the +// tenant's catalog, password line or authorization data, and the tenant's first +// query would fail on the member the Gateway just declared ready for it. +func (o *trinoPoolOperator) recordOneReceipt( + ctx context.Context, + binding trinoPoolTenantBinding, + current trinogateway.Publication, + basis trinoPoolBarrierBasis, + instances []configstore.TrinoPoolInstance, +) error { + byID := make(map[string]configstore.TrinoPoolInstance, len(instances)) + for _, instance := range instances { + byID[instance.InstanceID] = instance + } + + for _, instanceID := range current.MissingMembers { + instance, known := byID[instanceID] + if !known { + return fmt.Errorf("publication %s requires member %s, which this pool has no record of", + current.PublicationID, instanceID) + } + acknowledgement, err := o.acknowledge(ctx, instance, basis) + if err != nil { + slog.Info("Trino pool member is not serving the tenant's configuration yet.", + "pool", o.config.PublicID, "tenant", binding.Tenant, "instance", instanceID, "reason", err) + return nil + } + if acknowledgement.ProcessID != instance.CoordinatorBootID { + // The member restarted since it registered. Its receipt would be + // refused, and the Gateway's own rule is that a restart invalidates + // one: the health path will notice and replace it. + slog.Warn("Trino pool member restarted; its acknowledgement cannot be recorded.", + "pool", o.config.PublicID, "tenant", binding.Tenant, "instance", instanceID) + return nil + } + if _, err := o.gateway.RecordPublicationReceipt(ctx, o.config.RoutingGroup, current.PublicationID, + trinogateway.PublicationReceiptRequest{ + // The step identity carries the PUBLICATION, so a receipt for + // this attempt is a different operation from one for any other + // attempt against the same member. Without it the second + // attempt's receipt is the first one's step identity carrying a + // different applied revision, which the Gateway refuses as a + // changed intent - permanently, since the identity never moves + // again. One ordinary membership change during a deployment + // stranded a tenant that way. + Step: o.step("publication."+binding.Tenant, trinoPoolStepID("receipt", current.PublicationID, instanceID)), + InstanceID: instanceID, + PodUID: instance.CoordinatorPodUID, + BootID: instance.CoordinatorBootID, + AppliedRevision: current.TargetRevision, + AuthFingerprint: trinoPoolProjectionFingerprint(basis.Projection), + }); err != nil { + return o.dropAuthority(fmt.Errorf("record acknowledgement of %s for %s: %w", + instanceID, binding.Tenant, err)) + } + slog.Info("Trino pool member acknowledged a tenant's configuration.", + "pool", o.config.PublicID, "tenant", binding.Tenant, "instance", instanceID, + "target", current.TargetRevision) + return nil + } + return nil +} + +// commitBarrier closes the barrier, which is what actually admits the tenant. +func (o *trinoPoolOperator) commitBarrier( + ctx context.Context, + binding trinoPoolTenantBinding, + current trinogateway.Publication, + target string, + membershipGeneration int64, +) error { + var committed trinogateway.Publication + if err := o.runDurableStep(ctx, + configstore.TrinoPoolOperationSpec{ + OperationID: "publication:" + binding.Tenant + ":" + current.PublicationID, + PoolID: o.config.PoolID, + Kind: configstore.TrinoPoolOperationPublish, + IntentHash: trinoPoolPublicationPlanHash(binding, target), + }, + "commit", + trinoPoolCommitIntent{Tenant: binding.Tenant, Target: target, PublicationID: current.PublicationID}, + func(ctx context.Context) (string, error) { + result, err := o.gateway.CommitPublication(ctx, o.config.RoutingGroup, current.PublicationID, + trinogateway.CommitPublicationRequest{ + Step: o.step("publication."+binding.Tenant, trinoPoolStepID("commit", current.PublicationID)), + ExpectedMembershipGeneration: membershipGeneration, + }) + if err != nil { + return "", err + } + committed = result + return fmt.Sprintf(`{"phase":%q,"tenantState":%q}`, result.Phase, result.TenantState), nil + }, + ); err != nil { + return fmt.Errorf("commit publication for %s: %w", binding.Tenant, err) + } + if committed.Phase == "" { + // The step was recorded complete by an earlier attempt. The Gateway is + // authoritative for its own state, so the outcome is read back rather + // than assumed from the record. + current, err := o.gateway.GetPublication(ctx, o.config.RoutingGroup, current.PublicationID) + if err != nil { + return fmt.Errorf("read back publication for %s: %w", binding.Tenant, err) + } + committed = current + } + if committed.Phase != "ADMITTED" { + return fmt.Errorf("publication for %s did not admit the tenant: phase %s", binding.Tenant, committed.Phase) + } + return o.recordAdmitted(ctx, binding, target, committed) +} + +// recordAdmitted checkpoints an admission the Gateway has already made. The +// Gateway's record is authoritative from the moment it commits, so a failure +// here delays the checkpoint - it never retracts the admission. +func (o *trinoPoolOperator) recordAdmitted( + ctx context.Context, + binding trinoPoolTenantBinding, + target string, + publication trinogateway.Publication, +) error { + receipt, err := json.Marshal(map[string]any{ + "publicationId": publication.PublicationID, + "tenantState": publication.TenantState, + "receipts": len(publication.Receipts), + }) + if err != nil { + return fmt.Errorf("encode publication receipt for %s: %w", binding.Tenant, err) + } + slog.Info("Trino pool tenant admitted.", + "pool", o.config.PublicID, "tenant", binding.Tenant, "target", target, + "members", len(publication.Receipts)) + o.forgetBarrierBasis(publication.PublicationID) + return o.dropAuthority(o.publications.RecordTrinoPoolPublicationCommitted(ctx, o.lease, + o.config.PoolID, binding.Tenant, target, string(receipt))) +} + +// acknowledge asks ONE member what it is actually serving, against the +// configuration THIS attempt was opened at. +func (o *trinoPoolOperator) acknowledge( + ctx context.Context, + instance configstore.TrinoPoolInstance, + basis trinoPoolBarrierBasis, +) (trinoPoolAcknowledgement, error) { + if o.acknowledgement == nil { + return trinoPoolAcknowledgement{}, fmt.Errorf("this control plane cannot probe pool members") + } + if trinopool.Phase(instance.Phase) != trinopool.PhaseServing && trinopool.Phase(instance.Phase) != trinopool.PhaseAdmitted { + return trinoPoolAcknowledgement{}, fmt.Errorf("member %s is %s", instance.InstanceID, instance.Phase) + } + acknowledgement, err := o.acknowledgement(ctx, instance.EndpointURL, basis.Projection, basis.CatalogRevision) + if err != nil { + return trinoPoolAcknowledgement{}, err + } + if !acknowledgement.ProjectionCurrent { + return trinoPoolAcknowledgement{}, fmt.Errorf("member %s is not serving the current authorization and authentication projection", instance.InstanceID) + } + return acknowledgement, nil +} + +// rememberBarrierBasis captures the configuration an attempt is admitting +// against. There is at most one live attempt, so this holds one entry. +func (o *trinoPoolOperator) rememberBarrierBasis(publicationID string) { + if o.barrierBasis == nil { + o.barrierBasis = map[string]trinoPoolBarrierBasis{} + } + o.barrierBasis[publicationID] = trinoPoolBarrierBasis{ + Projection: o.expectedProjection(), + CatalogRevision: o.publishedCatalogRevision(), + } +} + +func (o *trinoPoolOperator) barrierBasisFor(publicationID string) (trinoPoolBarrierBasis, bool) { + basis, ok := o.barrierBasis[publicationID] + return basis, ok +} + +func (o *trinoPoolOperator) forgetBarrierBasis(publicationID string) { + delete(o.barrierBasis, publicationID) +} + +// tenantIsCurrent reports that this tenant's admission already covers its +// current intent. +// +// The intent is the tenant's OWN: the principal set it has now, admitted under +// the occurrence that is recorded for it. It deliberately does not include the +// pool's current catalog revision or the fleet-wide projection digest - those +// move every time ANY warehouse is provisioned or any login changes anywhere, +// which would re-admit the entire fleet for one new tenant. What a member must +// have applied is checked where it belongs: at receipt time, against that +// member, before its acknowledgement is recorded. +func (o *trinoPoolOperator) tenantIsCurrent( + binding trinoPoolTenantBinding, + publication configstore.TrinoPoolPublication, +) bool { + return publication.State == configstore.TrinoPublicationAdmitted && + publication.AdmittedTargetRevision != "" && + publication.PrincipalRevision == binding.Revision && + // The admitted target names the BINDING it admitted, and the occurrence + // that admitted it. Only the binding decides currency: the occurrence + // counter also moves for this tenant's publications and revocations, and + // comparing it would re-admit a tenant whose admission is perfectly + // current every time one of those happened. + strings.HasPrefix(publication.AdmittedTargetRevision, trinoPoolTargetPrefix(binding)) +} + +func (o *trinoPoolOperator) expectedProjection() trinoPoolProjectionRevisions { + if o.projection == nil { + return trinoPoolProjectionRevisions{} + } + return o.projection() +} + +func (o *trinoPoolOperator) publishedCatalogRevision() int64 { + if o.pool == nil { + return 0 + } + return o.pool.PublicationRevision +} + +// ensureCatalogWatermark makes the admission gate's revision authoritative +// before any tenant is published, admitted or certified against it. +// +// The failure this exists for: a catalog commits, the follow-up write of its +// revision onto the pool row fails, and NOTHING republishes it. That catalog +// already exists, so no later mutation carries the number forward; the gate +// keeps certifying members against a revision that predates the tenant, and a +// warehouse can be admitted - and reported ready - without its catalog. The +// tenant loop reads the enabled orgs on its own, so it would not even notice +// that a provisioner call had failed. +// +// Recovery is a bounded read of the store's own writer state, and it fails +// CLOSED: a watermark that cannot be read or cannot be checkpointed stops this +// tick's admissions rather than proceeding against a number nobody can +// confirm. The instance lifecycle is unaffected - reconcileOnce isolates this +// step's error - so a pool still repairs and drains while admissions hold. +func (o *trinoPoolOperator) ensureCatalogWatermark(ctx context.Context) error { + if o.catalogWatermark == nil { + // This cell publishes through a coordinator, which owns the catalog + // store itself. There is no duckgres-side authority to compare against, + // so the behaviour is exactly what it was. + return nil + } + published, err := o.catalogWatermark(ctx) + if err != nil { + return fmt.Errorf("read the published catalog revision for pool %s: %w", o.config.PublicID, err) + } + if o.pool == nil { + return fmt.Errorf("pool %s has no durable row to checkpoint against", o.config.PublicID) + } + if published <= o.pool.PublicationRevision { + return nil + } + // The row is behind the store. Checkpoint it under this term's authority + // before anything is certified against the stale value. + if err := o.store.RecordTrinoPoolPublicationRevision(ctx, o.lease, o.config.PoolID, published); err != nil { + return o.dropAuthority(fmt.Errorf("checkpoint the published catalog revision %d for pool %s: %w", + published, o.config.PublicID, err)) + } + slog.Warn("Trino pool recovered a catalog revision the publication never checkpointed.", + "pool", o.config.PublicID, "recorded", o.pool.PublicationRevision, "published", published) + o.pool.PublicationRevision = published + return nil +} + +// trinoPoolTargetRevision names ONE tenant's attempt: the principal set being +// admitted, and which occurrence is admitting it. +// +// The Gateway compares this string verbatim between the barrier and every +// receipt, so it identifies the attempt rather than describing the fleet. It is +// deliberately NOT a global configuration fingerprint: the catalog revision and +// the projection digest move whenever any warehouse anywhere is provisioned or +// changes a login, so a global target would expire every tenant's admission for +// somebody else's change and re-admit the whole fleet one five-second step at a +// time. +// +// What a member must actually be serving is checked against that member, at +// receipt time - it must have applied the published catalog revision and be +// deciding with the projection this attempt was opened at - which is strictly +// stronger than anything a name could assert, and costs nothing when nothing +// changed. +func trinoPoolTargetRevision(binding trinoPoolTenantBinding, attempt int64) string { + if attempt < 1 || binding.Revision == "" { + // No occurrence has been started for this tenant yet. Defaulting to one + // would make a tenant's FIRST barrier and its first post-revocation + // barrier share an identity, so the re-admission would replay the + // original attempt's recorded outcome instead of admitting the tenant + // again. + return "" + } + return fmt.Sprintf("%sa%d", trinoPoolTargetPrefix(binding), attempt) +} + +// trinoPoolTargetPrefix names the BINDING a target admits, exactly - the +// Gateway's revision alphabet takes 64 characters and a binding revision is 32, +// so nothing has to be truncated into an ambiguous prefix. +func trinoPoolTargetPrefix(binding trinoPoolTenantBinding) string { + return "b" + binding.Revision + "." +} + +// trinoPoolProjectionFingerprint is the 64-hex value the Gateway records with a +// receipt, derived from the projection the attempt was opened at, so an +// operator reading a receipt can tell which projection was acknowledged. +func trinoPoolProjectionFingerprint(projection trinoPoolProjectionRevisions) string { + digest := sha256.Sum256([]byte(strings.Join([]string{ + projection.Policy, projection.Password, projection.Group, + }, "\x00"))) + return hex.EncodeToString(digest[:]) +} + +// trinoPoolOccurrence names one durable occurrence inside a step identity. +func trinoPoolOccurrence(attempt int64) string { + return fmt.Sprintf("a%d", attempt) +} + +// trinoPoolStepID joins the parts of a step identity, within the Gateway's +// 64-character column. +// +// A step identity must be STABLE (a retry has to resolve the recorded outcome) +// and UNIQUE per intent (a different intent under one identity is refused +// forever). Truncating would break uniqueness silently, so an identity that +// does not fit keeps its leading parts and carries a digest of the rest: still +// deterministic, still unique, and short enough that the Gateway stores it. +func trinoPoolStepID(parts ...string) string { + const limit = 64 + joined := strings.Join(parts, ".") + if len(joined) <= limit { + return joined + } + digest := sha256.Sum256([]byte(joined)) + short := hex.EncodeToString(digest[:])[:24] + head := parts[0] + if len(head)+1+len(short) > limit { + head = head[:limit-1-len(short)] + } + return head + "." + short +} + +// trinoPoolPublicationID is deterministic in the tenant and the target, so a +// retry after a lost response addresses the same barrier instead of opening a +// second one. +func trinoPoolPublicationID(tenant, target string) string { + digest := sha256.Sum256([]byte(tenant + "\x00" + target)) + return "pub." + hex.EncodeToString(digest[:])[:24] +} + +// trinoPoolPublicationPlanHash is the barrier's immutable plan: this tenant, +// these principals, this configuration. +func trinoPoolPublicationPlanHash(binding trinoPoolTenantBinding, target string) string { + digest := sha256.New() + _, _ = digest.Write([]byte(binding.Tenant + "\x00" + binding.Revision + "\x00" + target + "\x00")) + for _, principal := range binding.Principals { + _, _ = digest.Write([]byte(principal)) + _, _ = digest.Write([]byte{0}) + } + return hex.EncodeToString(digest.Sum(nil)) +} + +// trinoPoolCommitIntent is what a commit MEANS, separate from the authority +// envelope that carries it. +type trinoPoolCommitIntent struct { + Tenant string + Target string + PublicationID string +} diff --git a/controlplane/trino_pool_publication_order_test.go b/controlplane/trino_pool_publication_order_test.go new file mode 100644 index 000000000..bfd1e793e --- /dev/null +++ b/controlplane/trino_pool_publication_order_test.go @@ -0,0 +1,102 @@ +//go:build kubernetes + +package controlplane + +import ( + "slices" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" +) + +// What duckgres has CHECKPOINTED for a tenant must always name the principal +// set the Gateway actually binds. +// +// The divergence this pins is subtle and permanent. A refused call is rolled +// back at the Gateway before anything is journalled, so the step identity stays +// unclaimed, and publishing principals carries no revision ordering of its own. +// So if a refusal of a REISSUE closed the occurrence, the controller could +// advance to a newer intent, publish it and checkpoint it - and then the copy +// that was still executing under the OLD occurrence lands and overwrites the +// Gateway's binding with the superseded set. duckgres believes it published the +// newer one and never sends it again: a removed login stays dispatchable and a +// current one cannot dispatch, until something else happens to move that +// tenant's binding. +// +// The property asserted here is the absence of that divergence, NOT that the +// newest desired set wins. Under the guard the tenant is deliberately HELD on +// its unresolved occurrence while an older copy may still be in flight, so the +// bound set may legitimately lag the desired one - see the limitation recorded +// in the shared-pool contract. What may never happen is the two records +// disagreeing about what is bound. +func TestACheckpointedBindingNeverDivergesFromTheGateway(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + tenants := &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.tenants = tenants + harness.fullMembership(t) + harness.admitAll(t, 1) + + // 1. The binding changes; the publication REACHES the Gateway and is left + // executing there while the caller's response is lost. + tenants.orgs[0].Users = []configstore.TrinoOrgUser{{Username: "engineer", PasswordHash: "hash"}} + harness.gateway.deferPublish = map[string]bool{"org-a": true} + for tick := 0; tick < 12 && harness.publications.rows["org-a"].PendingIntent == ""; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + if harness.publications.rows["org-a"].PendingIntent == "" { + t.Fatalf("no publication is in flight: %+v", harness.publications.rows["org-a"]) + } + + // 2. The REISSUE of that same occurrence is refused definitively - another + // tenant currently owns one of the identifiers. + harness.gateway.principalOf["acme.engineer"] = "org-old" + for tick := 0; tick < 12; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + // 3. The cause clears and the desired intent moves on again. + delete(harness.gateway.principalOf, "acme.engineer") + tenants.orgs[0].Users = []configstore.TrinoOrgUser{{Username: "analyst", PasswordHash: "hash"}} + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + // 4. The copy left executing in step 1 finally commits, after everything + // else, and the controller runs on. + harness.gateway.deliverDeferred() + for tick := 0; tick < 40; tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + + bound := harness.gateway.principals["org-a"] + checkpoint := harness.publications.rows["org-a"].PrincipalRevision + if checkpoint != trinoPoolBindingRevision(bound) { + t.Fatalf("duckgres has checkpointed revision %q while the Gateway binds %v (revision %q): "+ + "a delayed copy overwrote the binding and nothing will correct it", + checkpoint, bound, trinoPoolBindingRevision(bound)) + } + + // And once the request finally settles - here, the in-flight copy having + // landed IS its outcome - the tenant converges on the desired set rather + // than being stuck describing a stale one. + wanted := trinoPoolTenantBindingFor(tenants.orgs[0]) + for tick := 0; tick < 60 && !slices.Equal(harness.gateway.principals["org-a"], wanted.Principals); tick++ { + harness.tickTolerant(1) + harness.clearBackoff("org-a") + } + if got := harness.gateway.principals["org-a"]; !slices.Equal(got, wanted.Principals) { + t.Fatalf("the tenant never converged: bound %v, desired %v - the refusal cleared and the "+ + "delayed copy landed, so nothing is uncertain any more and the desired binding must be published", + got, wanted.Principals) + } + // Whatever it converged to, the two records must still agree. + bound = harness.gateway.principals["org-a"] + if got := harness.publications.rows["org-a"].PrincipalRevision; got != trinoPoolBindingRevision(bound) { + t.Fatalf("checkpoint %q disagrees with the bound set %v after settling", got, bound) + } +} diff --git a/controlplane/trino_pool_receipts.go b/controlplane/trino_pool_receipts.go new file mode 100644 index 000000000..699776aca --- /dev/null +++ b/controlplane/trino_pool_receipts.go @@ -0,0 +1,122 @@ +//go:build kubernetes + +package controlplane + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" +) + +// Receipts persisted alongside an instance. +// +// A receipt is EVIDENCE, kept so a later reader can answer "why was this +// admitted" or "on whose authority was this deleted" without re-deriving it. +// They are stored as JSON documents rather than scattered columns because they +// are read by people, not by queries. + +// storedValidationReceipt is the durable form of a candidate validation. +type storedValidationReceipt struct { + NodeID string `json:"nodeId"` + ProcessID string `json:"processId"` + CoordinatorID string `json:"coordinatorId"` + AppliedRevision int64 `json:"appliedRevision"` + AuthRevision string `json:"authRevision"` + ReadyWorkers int `json:"readyWorkers"` + Checks []string `json:"checks"` + CertificateHash string `json:"certificateHash"` + ObservedAt string `json:"observedAt"` + // Unacknowledged names security components that did not report what they + // loaded. It is persisted so an operator reading the receipt later can see + // exactly which part of the authorization state was never acknowledged, + // rather than inferring it from the absence of a check. + Unacknowledged []string `json:"unacknowledged,omitempty"` +} + +func marshalValidationReceipt(validation trinoPoolValidation) (string, error) { + encoded, err := json.Marshal(storedValidationReceipt{ + NodeID: validation.NodeID, + ProcessID: validation.ProcessID, + CoordinatorID: validation.CoordinatorID, + AppliedRevision: validation.AppliedRevision, + AuthRevision: validation.AuthRevision, + ReadyWorkers: validation.ReadyWorkers, + Checks: validation.Checks, + CertificateHash: validation.CertificateHash, + ObservedAt: nowUTC().Format(time.RFC3339), + Unacknowledged: validation.Unacknowledged, + }) + if err != nil { + return "", fmt.Errorf("encode validation receipt: %w", err) + } + return string(encoded), nil +} + +func unmarshalValidationReceipt(document string) (trinoPoolValidation, error) { + if document == "" || document == "{}" { + return trinoPoolValidation{}, fmt.Errorf("no validation receipt was recorded") + } + var stored storedValidationReceipt + if err := json.Unmarshal([]byte(document), &stored); err != nil { + return trinoPoolValidation{}, err + } + if stored.CertificateHash == "" || stored.ProcessID == "" { + return trinoPoolValidation{}, fmt.Errorf("the recorded validation receipt is incomplete") + } + return trinoPoolValidation{ + NodeID: stored.NodeID, + ProcessID: stored.ProcessID, + CoordinatorID: stored.CoordinatorID, + AppliedRevision: stored.AppliedRevision, + AuthRevision: stored.AuthRevision, + ReadyWorkers: stored.ReadyWorkers, + Checks: stored.Checks, + CertificateHash: stored.CertificateHash, + Unacknowledged: stored.Unacknowledged, + }, nil +} + +// storedRetirementReceipt records the Gateway's irreversible claim. It is +// written BEFORE any resource is deleted, so an interrupted deletion can always +// be resumed with proof that the claim existed. +type storedRetirementReceipt struct { + Incarnation string `json:"incarnation"` + Phase string `json:"phase"` + Generation int64 `json:"generation"` + RetirementKind string `json:"retirementKind"` + ClaimedAt string `json:"claimedAt"` +} + +func marshalRetirementReceipt(member trinogateway.Member) (string, error) { + encoded, err := json.Marshal(storedRetirementReceipt{ + Incarnation: member.Incarnation, + Phase: member.Phase, + Generation: member.Generation, + RetirementKind: member.RetirementKind, + ClaimedAt: nowUTC().Format(time.RFC3339), + }) + if err != nil { + return "", fmt.Errorf("encode retirement receipt: %w", err) + } + return string(encoded), nil +} + +// instanceNamespace reports where an instance's objects live. The namespace is +// the blueprint's and is pinned per instance, so a pool that is later moved +// cannot make an old instance's delete target the wrong namespace. +func instanceNamespace(instance configstore.TrinoPoolInstance) string { + var snapshot struct { + Namespace string `json:"namespace"` + } + if err := json.Unmarshal([]byte(instance.BlueprintSnapshot), &snapshot); err != nil { + return "" + } + return snapshot.Namespace +} + +// nowUTC exists so tests can observe stable timestamps without reaching for a +// global clock. +var nowUTC = func() time.Time { return time.Now().UTC() } diff --git a/controlplane/trino_pool_validate.go b/controlplane/trino_pool_validate.go new file mode 100644 index 000000000..e0db68720 --- /dev/null +++ b/controlplane/trino_pool_validate.go @@ -0,0 +1,582 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "sort" + "strings" + "time" + + "github.com/posthog/duckgres/controlplane/provisioner" + "github.com/posthog/duckgres/controlplane/trinogateway" +) + +// Candidate validation: what duckgres checks about a PREPARING instance before +// asking the Gateway to admit it. +// +// There is no canary warehouse, no canary credential and no synthetic tenant. +// The checks reuse the operational credential the provisioner already holds and +// the coordinator's own authenticated endpoints. Two rules shape this file: +// +// - The candidate is probed through ITS OWN Service, never through the pool's +// load-balanced endpoint. A pooled probe can silently validate a different +// cluster and admit an unready one. +// - A check that was not performed is not reported. The receipt lists exactly +// what was observed, because the Gateway records that list verbatim and an +// operator will read it as evidence. +const ( + // A structural check, not a workload: one request budget, one pass budget. + trinoPoolProbeRequestBudget = 10 * time.Second + trinoPoolProbePassBudget = 5 * time.Minute + catalogSyncPath = "/v1/catalog/sync" +) + +var errTrinoPoolCandidateNotReady = errors.New("trino pool candidate is not ready") + +// The checks duckgres reports on an admission receipt. They mirror the +// constants in the Gateway client and are listed verbatim in the Gateway's +// record, so each one must correspond to something actually observed above. +const ( + trinoPoolCheckImage = trinogateway.CheckImage + trinoPoolCheckWorkers = trinogateway.CheckWorkers + trinoPoolCheckCatalogRevision = trinogateway.CheckCatalogRevision + trinoPoolCheckAuthRevision = trinogateway.CheckAuthRevision + trinoPoolCheckOperationalConnection = trinogateway.CheckOperationalConnection +) + +// catalogSyncStatus is the coordinator's /v1/catalog/sync response. +// +// observedRevision and appliedRevision are null until the first successful +// snapshot, which is why they are pointers: zero would be indistinguishable +// from "revision 0 applied", and revision 0 is the legitimate empty state. +type catalogSyncStatus struct { + NodeID string `json:"nodeId"` + NodeVersion string `json:"nodeVersion"` + ProcessID string `json:"processId"` + CoordinatorID string `json:"coordinatorId"` + Enabled bool `json:"enabled"` + Ready bool `json:"ready"` + ObservedRevision *int64 `json:"observedRevision"` + AppliedRevision *int64 `json:"appliedRevision"` + ActiveCatalogs int `json:"activeCatalogs"` + FailedCatalogs int `json:"failedCatalogs"` + LastFailure string `json:"lastFailure"` + NotReadyReason string `json:"notReadyReason"` + SecurityRevisions []componentRevision `json:"securityRevisions"` +} + +// componentRevision is what one security component of the process has loaded. +// This is the coordinator's answer to "which credentials, groups and +// authorization data are actually in effect", which a catalog revision says +// nothing about. +type componentRevision struct { + Kind string `json:"kind"` + Name string `json:"name"` + Revision string `json:"revision"` + Error string `json:"error"` +} + +// trinoPoolExpectation is what the pool requires of a candidate. Every field is +// compared; none of them is assumed. +type trinoPoolExpectation struct { + // Image is the blueprint's digest-pinned release image. + Image string + // CatalogRevision is the pool's published catalog revision. A structurally + // healthy coordinator sitting at an older revision is NOT certified: it + // would serve a catalog set that does not include the newest tenant. + CatalogRevision int64 + // PolicyRevision is the authorization projection this control plane + // currently serves. The candidate's access control must report deciding + // with exactly this value before the auth-revision check may be claimed. + // Empty means the projection is unknown here, and nothing may be claimed on + // its behalf - which fails admission closed at the Gateway. + PolicyRevision string + // ProjectionDigest is the ACCEPTED projection: the authorization bundle's + // revision and the two authentication-file fingerprints, as one value, read + // from the control plane's durable record rather than from this process's + // memory of what it last published. + ProjectionDigest string + // PasswordRevision and GroupRevision are the fingerprints of the + // authentication files this control plane has projected. + // + // They are checked for the same reason as the policy revision, and they are + // NOT implied by it: the OPA bundle and the auth Secret reach a coordinator + // by different paths and at different times, so a candidate can be deciding + // with the current authorization data while its password store still + // predates the tenant that is about to be admitted. That candidate passes an + // authorization-only check and then rejects that tenant's very first + // request. + PasswordRevision string + GroupRevision string + // InternalHTTP marks a pooled coordinator reached on its in-cluster + // Service, where TLS terminates at the Gateway. + InternalHTTP bool +} + +// trinoPoolValidation is a process-bound validation result. +type trinoPoolValidation struct { + NodeID string + ProcessID string + CoordinatorID string + AppliedRevision int64 + AuthRevision string + ReadyWorkers int + Checks []string + CertificateHash string + // Unacknowledged names the security components that did NOT report a + // loaded revision. It is recorded and surfaced rather than being folded + // into a pass: a component that cannot say what it loaded has not + // acknowledged anything, and treating its silence as agreement is exactly + // the false readiness this validation exists to prevent. + Unacknowledged []string +} + +// validateTrinoPoolCandidate probes one candidate through its own endpoint. +// +// coordinatorURL must be the instance's own Service. readyWorkers comes from +// the Kubernetes observation, and the coordinator's own node inventory has to +// agree with it: a coordinator that reports fewer registered workers than the +// cluster has running pods is still warming up, and admitting it would send +// tenant queries to a cluster that cannot plan them. +func validateTrinoPoolCandidate( + ctx context.Context, + client *http.Client, + coordinatorURL string, + credential func() (string, string), + observed trinoPoolObservation, + expected trinoPoolExpectation, +) (trinoPoolValidation, error) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolProbePassBudget) + defer cancel() + + // The image is checked against what the cluster is RUNNING, not against + // what the spec asked for. Claiming the check without comparing anything + // put a false acknowledgement into the Gateway's durable evidence - the + // same defect the auth-revision handling exists to avoid. + if expected.Image == "" { + return trinoPoolValidation{}, fmt.Errorf("%w: no expected image to verify against", errTrinoPoolCandidateNotReady) + } + if observed.CoordinatorImage != expected.Image { + return trinoPoolValidation{}, fmt.Errorf("%w: coordinator runs image %q, the release pins %q", + errTrinoPoolCandidateNotReady, observed.CoordinatorImage, expected.Image) + } + if observed.WorkerImage != "" && observed.WorkerImage != expected.Image { + return trinoPoolValidation{}, fmt.Errorf("%w: workers run image %q, the release pins %q", + errTrinoPoolCandidateNotReady, observed.WorkerImage, expected.Image) + } + + observedWorkers := observed.ReadyWorkers + requiredCatalogRevision := expected.CatalogRevision + + username, password := credential() + sql := rolloutSQLClient{ + baseURL: coordinatorURL, client: client, username: username, password: password, + internalHTTP: expected.InternalHTTP, + } + + before, err := sql.info(ctx) + if err != nil { + return trinoPoolValidation{}, fmt.Errorf("%w: %v", errTrinoPoolCandidateNotReady, err) + } + + sync, err := fetchCatalogSync(ctx, client, coordinatorURL, username, password, expected.InternalHTTP) + if err != nil { + return trinoPoolValidation{}, err + } + if !sync.Enabled { + return trinoPoolValidation{}, fmt.Errorf("%w: catalog synchronization is disabled on the candidate", errTrinoPoolCandidateNotReady) + } + if !sync.Ready || sync.FailedCatalogs != 0 { + return trinoPoolValidation{}, fmt.Errorf("%w: %s", errTrinoPoolCandidateNotReady, syncReason(sync)) + } + if sync.AppliedRevision == nil || *sync.AppliedRevision < requiredCatalogRevision { + // Admitting a member behind the published revision would serve a tenant + // a catalog set that does not include them yet. + return trinoPoolValidation{}, fmt.Errorf("%w: applied catalog revision %v is behind the published revision %d", + errTrinoPoolCandidateNotReady, revisionText(sync.AppliedRevision), requiredCatalogRevision) + } + // The process identity has to be the one we started talking to. A restart + // mid-validation invalidates everything observed before it. + if sync.NodeID != before.NodeID || (sync.CoordinatorID != "" && sync.CoordinatorID != before.CoordinatorID) { + return trinoPoolValidation{}, fmt.Errorf("%w: coordinator identity changed during validation", errTrinoPoolCandidateNotReady) + } + if sync.ProcessID == "" { + return trinoPoolValidation{}, fmt.Errorf("%w: candidate reports no process identity", errTrinoPoolCandidateNotReady) + } + + registered, err := registeredWorkerCount(ctx, sql, before.NodeID) + if err != nil { + return trinoPoolValidation{}, err + } + if observedWorkers == 0 || registered != observedWorkers { + return trinoPoolValidation{}, fmt.Errorf("%w: %d workers registered, %d running", + errTrinoPoolCandidateNotReady, registered, observedWorkers) + } + + // Re-read the identity last: anything observed above is only valid if the + // same process was serving throughout. + after, err := sql.info(ctx) + if err != nil || after.NodeID != before.NodeID || after.CoordinatorID != before.CoordinatorID { + return trinoPoolValidation{}, fmt.Errorf("%w: coordinator changed during observation", errTrinoPoolCandidateNotReady) + } + + acknowledged, unacknowledged := splitSecurityRevisions(sync.SecurityRevisions) + checks := []string{ + trinoPoolCheckImage, + trinoPoolCheckWorkers, + trinoPoolCheckCatalogRevision, + trinoPoolCheckOperationalConnection, + } + // The auth-revision check is claimed ONLY when both of these hold: + // + // - every security component the coordinator exposes reported what it + // loaded, and + // - the authorization data one of them reports is the projection THIS + // control plane is serving right now. + // + // The second condition is the one that makes the check mean anything. "The + // components answered" proves a coordinator can describe itself, not that + // it decides with current data: a pooled coordinator whose OPA still serves + // the bundle from before a tenant was provisioned answers perfectly and + // authorizes against a policy that has never heard of that tenant. The + // Gateway records this list verbatim and an operator reads it as evidence, + // so an unverifiable claim must be absent rather than optimistic. + // + // It requires `opa.policy.revision-uri` on a pooled coordinator, pointed at + // the document the bundle publishes. Without it the access control reports + // nothing, the check is absent, and admission fails closed. + // Every component whose data this control plane projects must report having + // loaded exactly what is being served: the authorization bundle, the + // password file and the group file. They travel by different paths and + // settle at different times, so one being current says nothing about the + // others - a coordinator with the newest bundle and a password file from + // before the tenant existed refuses that tenant's first request while + // looking perfectly healthy. + // + // The comparison is against the ACCEPTED projection - the control plane's + // durable record - not against whatever this process last projected. A + // replica's own memory says what IT published, which is exactly the thing + // in question when replicas disagree. + projectionAcknowledged := expected.ProjectionDigest != "" && + reportedProjectionDigest(sync.SecurityRevisions) == expected.ProjectionDigest + if len(acknowledged) > 0 && len(unacknowledged) == 0 && projectionAcknowledged { + checks = append(checks, trinoPoolCheckAuthRevision) + } + + validation := trinoPoolValidation{ + NodeID: sync.NodeID, + ProcessID: sync.ProcessID, + CoordinatorID: before.CoordinatorID, + AppliedRevision: *sync.AppliedRevision, + AuthRevision: authRevisionFingerprint(sync.SecurityRevisions), + ReadyWorkers: registered, + Checks: checks, + Unacknowledged: unacknowledged, + } + validation.CertificateHash = certificateHash(validation) + return validation, nil +} + +// trinoAccessControlKind is how the coordinator names an authorization +// component in its readiness report. Matching on the KIND rather than on the +// configured implementation name keeps this working for a deployment that names +// its access control something other than "opa". +const ( + trinoAccessControlKind = "system-access-control" + trinoPasswordAuthenticator = "password-authenticator" + trinoGroupProviderComponent = "group-provider" +) + +// reportedProjectionDigest names the projection a coordinator says it is +// deciding and authenticating with, in the SAME form the control plane names +// what it published. +// +// It is empty - matching nothing - unless each required component is present +// exactly once and reported a revision. A second password authenticator reads a +// file this control plane does not write, so it could authenticate principals +// outside the projection; two disagreeing answers are not a projection. +func reportedProjectionDigest(revisions []componentRevision) string { + reported := map[string]string{} + for _, revision := range revisions { + switch revision.Kind { + case trinoAccessControlKind, trinoPasswordAuthenticator, trinoGroupProviderComponent: + default: + continue + } + if revision.Error != "" || strings.TrimSpace(revision.Revision) == "" { + return "" + } + if existing, seen := reported[revision.Kind]; seen && existing != revision.Revision { + return "" + } + reported[revision.Kind] = revision.Revision + } + return provisioner.TrinoProjectionDigest( + reported[trinoAccessControlKind], + reported[trinoPasswordAuthenticator], + reported[trinoGroupProviderComponent], + ) +} + +// reportsRevision reports whether this kind of component is present AND every +// instance of it acknowledged exactly this revision. +// +// Equality, not ordering: the question is whether the coordinator decides with +// the data being served, and a coordinator carrying a LATER revision than this +// replica knows about is equally uncertifiable here. +// +// EVERY instance has to match, not merely one. A coordinator configured with a +// second password authenticator - a file this control plane does not write - +// can authenticate principals outside the projection, and admitting it on the +// strength of the one component that agrees would put that file inside the +// pool's trust boundary without anybody stating it. +func reportsRevision(revisions []componentRevision, kind, revision string) bool { + found := false + for _, reported := range revisions { + if reported.Kind != kind { + continue + } + if reported.Error != "" || reported.Revision != revision { + return false + } + found = true + } + return found +} + +// authRevisionFingerprint condenses what the process's security components have +// loaded into one opaque value. +// +// It is a hash, not the revisions themselves: the value is sent to the Gateway +// and shown to operators, and component revisions can carry credential-derived +// data. A component reporting an ERROR is folded in, so a coordinator whose +// password provider failed to load produces a different fingerprint from one +// where it loaded cleanly — the two must never compare equal. +func authRevisionFingerprint(revisions []componentRevision) string { + if len(revisions) == 0 { + // Nothing to acknowledge. Reported as an explicit marker rather than an + // empty string, which the Gateway rejects and which would read as "no + // auth configured" instead of "this build reports nothing". + return "none" + } + ordered := make([]string, 0, len(revisions)) + for _, revision := range revisions { + ordered = append(ordered, strings.Join([]string{revision.Kind, revision.Name, revision.Revision, revision.Error}, "\x00")) + } + sort.Strings(ordered) + digest := sha256.Sum256([]byte(strings.Join(ordered, "\x1e"))) + return hex.EncodeToString(digest[:]) +} + +// certificateHash binds the receipt to the exact facts it asserts, so a +// certificate cannot be reused for a different process or revision. +func certificateHash(validation trinoPoolValidation) string { + digest := sha256.Sum256([]byte(strings.Join([]string{ + validation.NodeID, + validation.ProcessID, + validation.CoordinatorID, + fmt.Sprint(validation.AppliedRevision), + validation.AuthRevision, + fmt.Sprint(validation.ReadyWorkers), + strings.Join(validation.Checks, ","), + }, "\x00"))) + return hex.EncodeToString(digest[:]) +} + +func fetchCatalogSync(ctx context.Context, client *http.Client, coordinatorURL, username, password string, internalHTTP bool) (catalogSyncStatus, error) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolProbeRequestBudget) + defer cancel() + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, coordinatorURL+catalogSyncPath, nil) + if err != nil { + return catalogSyncStatus{}, fmt.Errorf("build catalog sync request: %w", err) + } + request.SetBasicAuth(username, password) + request.Header.Set("Accept", "application/json") + if internalHTTP { + // Same forwarded-HTTPS declaration the statement client sends: the + // readiness endpoint is authenticated too, and a management probe must + // not be the one path that quietly drops that requirement. + request.Header.Set("X-Forwarded-Proto", forwardedScheme) + request.Header.Set("X-Forwarded-Port", "443") + } + response, err := client.Do(request) + if err != nil { + return catalogSyncStatus{}, fmt.Errorf("%w: catalog sync request failed", errTrinoPoolCandidateNotReady) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return catalogSyncStatus{}, fmt.Errorf("%w: catalog sync returned HTTP %d", errTrinoPoolCandidateNotReady, response.StatusCode) + } + var status catalogSyncStatus + if err := json.NewDecoder(response.Body).Decode(&status); err != nil { + return catalogSyncStatus{}, fmt.Errorf("%w: catalog sync response is unreadable", errTrinoPoolCandidateNotReady) + } + return status, nil +} + +// registeredWorkerCount reads the coordinator's own node inventory and counts +// the active workers that have registered with it. +func registeredWorkerCount(ctx context.Context, sql rolloutSQLClient, coordinatorNodeID string) (int, error) { + rows, err := sql.statement(ctx, "SELECT node_id, coordinator, state FROM system.runtime.nodes") + if err != nil { + return 0, fmt.Errorf("%w: node inventory unavailable", errTrinoPoolCandidateNotReady) + } + workers, coordinators := 0, 0 + for _, row := range rows { + if len(row) != 3 { + return 0, fmt.Errorf("%w: invalid node inventory row", errTrinoPoolCandidateNotReady) + } + nodeID, idOK := row[0].(string) + isCoordinator, coordinatorOK := row[1].(bool) + state, stateOK := row[2].(string) + if !idOK || !coordinatorOK || !stateOK || state != "active" { + return 0, fmt.Errorf("%w: inactive or invalid node in the inventory", errTrinoPoolCandidateNotReady) + } + if isCoordinator { + coordinators++ + if nodeID != coordinatorNodeID { + return 0, fmt.Errorf("%w: the inventory names a different coordinator", errTrinoPoolCandidateNotReady) + } + continue + } + workers++ + } + if coordinators != 1 { + return 0, fmt.Errorf("%w: %d coordinators in the inventory", errTrinoPoolCandidateNotReady, coordinators) + } + return workers, nil +} + +func syncReason(status catalogSyncStatus) string { + switch { + case status.NotReadyReason != "": + return status.NotReadyReason + case status.LastFailure != "": + return status.LastFailure + case status.FailedCatalogs != 0: + return fmt.Sprintf("%d catalogs failed to apply", status.FailedCatalogs) + default: + return "candidate reports itself not ready" + } +} + +func revisionText(revision *int64) string { + if revision == nil { + return "none" + } + return fmt.Sprint(*revision) +} + +// splitSecurityRevisions separates components that reported a loaded revision +// from those that did not. +// +// A component reports an error when it cannot describe its own loaded state - +// for the OPA access control that is the normal case today, because the plugin +// does not implement the interface that would let it say which bundle revision +// its decisions are being made against. That silence is a fact about the +// system, not a validation failure to retry, so it is carried forward rather +// than swallowed. +func splitSecurityRevisions(revisions []componentRevision) (acknowledged, unacknowledged []string) { + for _, revision := range revisions { + name := revision.Kind + "/" + revision.Name + if revision.Error != "" || strings.TrimSpace(revision.Revision) == "" { + unacknowledged = append(unacknowledged, name) + continue + } + acknowledged = append(acknowledged, name) + } + sort.Strings(acknowledged) + sort.Strings(unacknowledged) + return acknowledged, unacknowledged +} + +// probeProcessIdentity reads ONLY the coordinator's process identity. +// +// It runs before member registration because the Gateway binds podUid and +// bootId at registration and then requires the admission receipt to carry the +// identical pair. Registering the pod UID as the boot id and admitting with the +// Trino processId made every admission fail POOL_NOT_CERTIFIED: there is one +// authoritative boot identity, and it is the coordinator's processId, which +// changes on every JVM start exactly as a boot identity must. +func probeProcessIdentity(ctx context.Context, client *http.Client, coordinatorURL string, credential func() (string, string), internalHTTP bool) (string, error) { + ctx, cancel := context.WithTimeout(ctx, trinoPoolProbeRequestBudget) + defer cancel() + + username, password := credential() + status, err := fetchCatalogSync(ctx, client, coordinatorURL, username, password, internalHTTP) + if err != nil { + return "", err + } + if status.ProcessID == "" { + return "", fmt.Errorf("%w: candidate reports no process identity", errTrinoPoolCandidateNotReady) + } + return status.ProcessID, nil +} + +// trinoPoolProjectionRevisions is what this control plane currently serves to +// coordinators: the authorization bundle's revision and the fingerprints of the +// password and group files. +// +// They are carried together because they are checked together. Any one of them +// being current is not evidence about the others: the bundle is pulled over +// HTTP on OPA's schedule, while the files arrive as a mounted Secret the +// kubelet refreshes on its own. +type trinoPoolProjectionRevisions struct { + Policy string + Password string + Group string +} + +// probeMemberAcknowledgement asks one serving member what configuration it is +// actually serving. +// +// It is the same authenticated readiness endpoint candidate validation uses, +// and deliberately so: a publication receipt asserts that this member will +// serve the tenant correctly, and the only evidence for that is what the member +// reports having loaded. Asserting it from what the controller published would +// commit a barrier while a coordinator still lacked the tenant's catalog, +// password line or authorization data. +func probeMemberAcknowledgement( + ctx context.Context, + client *http.Client, + coordinatorURL string, + credential func() (string, string), + expected trinoPoolProjectionRevisions, + catalogRevision int64, +) (trinoPoolAcknowledgement, error) { + username, password := credential() + sync, err := fetchCatalogSync(ctx, client, coordinatorURL, username, password, true) + if err != nil { + return trinoPoolAcknowledgement{}, err + } + if !sync.Enabled || !sync.Ready || sync.FailedCatalogs != 0 { + return trinoPoolAcknowledgement{}, fmt.Errorf("%w: %s", errTrinoPoolCandidateNotReady, syncReason(sync)) + } + if sync.ProcessID == "" { + return trinoPoolAcknowledgement{}, fmt.Errorf("%w: member reports no process identity", errTrinoPoolCandidateNotReady) + } + if sync.AppliedRevision == nil || *sync.AppliedRevision < catalogRevision { + // The member has not applied the catalog set that contains this tenant, + // so acknowledging on its behalf would admit the tenant to a coordinator + // that cannot resolve its catalog. + return trinoPoolAcknowledgement{}, fmt.Errorf("%w: applied catalog revision %v is behind the published revision %d", + errTrinoPoolCandidateNotReady, revisionText(sync.AppliedRevision), catalogRevision) + } + current := expected.Policy != "" && expected.Password != "" && expected.Group != "" && + reportsRevision(sync.SecurityRevisions, trinoAccessControlKind, expected.Policy) && + reportsRevision(sync.SecurityRevisions, trinoPasswordAuthenticator, expected.Password) && + reportsRevision(sync.SecurityRevisions, trinoGroupProviderComponent, expected.Group) + return trinoPoolAcknowledgement{ + ProcessID: sync.ProcessID, + AppliedRevision: *sync.AppliedRevision, + ProjectionCurrent: current, + }, nil +} diff --git a/controlplane/trino_pool_validate_test.go b/controlplane/trino_pool_validate_test.go new file mode 100644 index 000000000..9b529bbc4 --- /dev/null +++ b/controlplane/trino_pool_validate_test.go @@ -0,0 +1,510 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/posthog/duckgres/controlplane/provisioner" +) + +type fakeCoordinator struct { + server *httptest.Server + info map[string]any + sync map[string]any + nodes [][]any + requests []string +} + +func newFakeCoordinator(t *testing.T) *fakeCoordinator { + t.Helper() + coordinator := &fakeCoordinator{ + info: map[string]any{"coordinator": true, "starting": false, "nodeId": "node-1", "coordinatorId": "abcde"}, + sync: map[string]any{ + "nodeId": "node-1", "nodeVersion": "484", "processId": "process-1", "coordinatorId": "abcde", + "enabled": true, "ready": true, "observedRevision": 42, "appliedRevision": 42, + "activeCatalogs": 7, "failedCatalogs": 0, + "securityRevisions": []any{ + map[string]any{"kind": "password-authenticator", "name": "file", "revision": fakePasswordRevision}, + map[string]any{"kind": "group-provider", "name": "file", "revision": fakeGroupRevision}, + // The authorization projection the coordinator's OPA reports + // deciding with. This is the value the controller compares + // against what it currently serves. + map[string]any{"kind": "system-access-control", "name": "opa", "revision": fakePolicyRevision}, + }, + }, + nodes: [][]any{ + {"node-1", true, "active"}, + {"worker-1", false, "active"}, + {"worker-2", false, "active"}, + }, + } + // TLS, because the probe refuses a plaintext coordinator endpoint: it + // carries an operational credential. + coordinator.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + coordinator.requests = append(coordinator.requests, r.URL.Path) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v1/info": + _ = json.NewEncoder(w).Encode(coordinator.info) + case "/v1/catalog/sync": + _ = json.NewEncoder(w).Encode(coordinator.sync) + case "/v1/statement": + _ = json.NewEncoder(w).Encode(map[string]any{"data": coordinator.nodes}) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(coordinator.server.Close) + return coordinator +} + +// fakePolicyRevision stands in for the authorization projection this control +// plane serves. Its exact shape does not matter; that the two sides compare +// EQUAL does. +// The fingerprints of the authentication files this control plane projects, +// in the form Trino's file components publish. +const ( + fakePasswordRevision = "sha256:00000000000000000000000000000000000000000000000000000000000000aa" + fakeGroupRevision = "sha256:00000000000000000000000000000000000000000000000000000000000000bb" +) + +const fakePolicyRevision = "v2.0000000000000000000000000000000000000000000000000000000000000001" + +const fakeCoordinatorImage = "registry.example.invalid/trino@sha256:1111111111111111111111111111111111111111111111111111111111111111" + +func (c *fakeCoordinator) validate(t *testing.T, observedWorkers int, requiredRevision int64) (trinoPoolValidation, error) { + t.Helper() + return c.validateWith(t, trinoPoolObservation{ + ReadyWorkers: observedWorkers, DesiredWorkers: observedWorkers, + CoordinatorReady: true, CoordinatorPodUID: "pod-uid", + CoordinatorImage: fakeCoordinatorImage, WorkerImage: fakeCoordinatorImage, + }, trinoPoolExpectation{ + Image: fakeCoordinatorImage, CatalogRevision: requiredRevision, + // The ACCEPTED projection, as the durable record names it. + ProjectionDigest: provisioner.TrinoProjectionDigest( + fakePolicyRevision, fakePasswordRevision, fakeGroupRevision), + InternalHTTP: false, + }) +} + +func (c *fakeCoordinator) validateWith(t *testing.T, observed trinoPoolObservation, expected trinoPoolExpectation) (trinoPoolValidation, error) { + t.Helper() + return validateTrinoPoolCandidate( + context.Background(), + c.server.Client(), + c.server.URL, + func() (string, string) { return "observer", "secret" }, + observed, + expected, + ) +} + +func TestValidateCandidateAcceptsAReadyCoordinator(t *testing.T) { + coordinator := newFakeCoordinator(t) + validation, err := coordinator.validate(t, 2, 42) + if err != nil { + t.Fatalf("validate: %v", err) + } + if validation.NodeID != "node-1" || validation.ProcessID != "process-1" || validation.CoordinatorID != "abcde" { + t.Fatalf("validation identity = %+v", validation) + } + if validation.ReadyWorkers != 2 || validation.AppliedRevision != 42 { + t.Fatalf("validation = %+v", validation) + } + if validation.AuthRevision == "" || validation.CertificateHash == "" { + t.Fatal("the receipt carries no auth revision or certificate hash") + } + if len(validation.Checks) != 5 { + t.Fatalf("checks = %v", validation.Checks) + } +} + +// A coordinator with a healthy HTTP endpoint is not a ready member. These are +// the cases where /v1/info alone would have admitted a cluster that cannot +// serve. +func TestValidateCandidateRejects(t *testing.T) { + cases := map[string]func(*fakeCoordinator){ + "catalog sync disabled": func(c *fakeCoordinator) { c.sync["enabled"] = false }, + "not ready": func(c *fakeCoordinator) { c.sync["ready"] = false; c.sync["notReadyReason"] = "still applying" }, + "failed catalogs": func(c *fakeCoordinator) { + c.sync["failedCatalogs"] = 2 + }, + "no revision applied yet": func(c *fakeCoordinator) { c.sync["appliedRevision"] = nil }, + "revision behind the published one": func(c *fakeCoordinator) { + c.sync["appliedRevision"] = 41 + c.sync["observedRevision"] = 41 + }, + "no process identity": func(c *fakeCoordinator) { c.sync["processId"] = "" }, + "identity changed mid-validation": func(c *fakeCoordinator) { + c.sync["nodeId"] = "node-2" + }, + // Zero registered workers with a healthy coordinator is the classic + // "looks up, cannot plan a query" state. + "no workers registered": func(c *fakeCoordinator) { + c.nodes = [][]any{{"node-1", true, "active"}} + }, + "a worker is not active": func(c *fakeCoordinator) { + c.nodes = [][]any{{"node-1", true, "active"}, {"worker-1", false, "shutting_down"}} + }, + "the inventory names another coordinator": func(c *fakeCoordinator) { + c.nodes = [][]any{{"node-9", true, "active"}, {"worker-1", false, "active"}} + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + coordinator := newFakeCoordinator(t) + mutate(coordinator) + if _, err := coordinator.validate(t, 2, 42); !errors.Is(err, errTrinoPoolCandidateNotReady) { + t.Fatalf("error = %v, want errTrinoPoolCandidateNotReady", err) + } + }) + } +} + +// The count the coordinator reports must agree with the pods Kubernetes is +// running. A mismatch means the cluster is still converging. +func TestValidateCandidateRequiresWorkerCountsToAgree(t *testing.T) { + coordinator := newFakeCoordinator(t) + if _, err := coordinator.validate(t, 4, 42); !errors.Is(err, errTrinoPoolCandidateNotReady) { + t.Fatalf("error = %v, want a mismatch to be rejected", err) + } +} + +// The auth revision is reported as an opaque fingerprint: component revisions +// can carry credential-derived data, and this value is sent to the Gateway and +// shown to operators. +func TestAuthRevisionFingerprintIsOpaqueAndSensitive(t *testing.T) { + base := authRevisionFingerprint([]componentRevision{ + {Kind: "password-authenticator", Name: "file", Revision: "9"}, + {Kind: "group-provider", Name: "file", Revision: "4"}, + }) + reordered := authRevisionFingerprint([]componentRevision{ + {Kind: "group-provider", Name: "file", Revision: "4"}, + {Kind: "password-authenticator", Name: "file", Revision: "9"}, + }) + if base != reordered { + t.Fatal("the fingerprint depends on component ordering") + } + changed := authRevisionFingerprint([]componentRevision{ + {Kind: "password-authenticator", Name: "file", Revision: "10"}, + {Kind: "group-provider", Name: "file", Revision: "4"}, + }) + if changed == base { + t.Fatal("a changed component revision produced the same fingerprint") + } + // A component that FAILED to load must never fingerprint the same as one + // that loaded cleanly at the same revision. + failed := authRevisionFingerprint([]componentRevision{ + {Kind: "password-authenticator", Name: "file", Revision: "9", Error: "cannot read password.db"}, + {Kind: "group-provider", Name: "file", Revision: "4"}, + }) + if failed == base { + t.Fatal("a failed component fingerprinted as a healthy one") + } + if len(base) != 64 { + t.Fatalf("fingerprint %q is not opaque", base) + } + if authRevisionFingerprint(nil) != "none" { + t.Fatal("an absent revision set must be reported explicitly") + } +} + +// The certificate binds the receipt to the facts it asserts, so it cannot be +// replayed for a different process or a different revision. +func TestCertificateHashBindsTheObservedFacts(t *testing.T) { + base := trinoPoolValidation{ + NodeID: "node-1", ProcessID: "process-1", CoordinatorID: "abcde", + AppliedRevision: 42, AuthRevision: "auth", ReadyWorkers: 2, + Checks: []string{trinoPoolCheckImage}, + } + hash := certificateHash(base) + for name, mutate := range map[string]func(*trinoPoolValidation){ + "process": func(v *trinoPoolValidation) { v.ProcessID = "process-2" }, + "revision": func(v *trinoPoolValidation) { v.AppliedRevision = 43 }, + "auth": func(v *trinoPoolValidation) { v.AuthRevision = "other" }, + "workers": func(v *trinoPoolValidation) { v.ReadyWorkers = 3 }, + } { + changed := base + mutate(&changed) + if certificateHash(changed) == hash { + t.Errorf("changing the %s did not change the certificate hash", name) + } + } +} + +// A component that cannot report what it loaded has acknowledged nothing. +// Trino's file password authenticator and group provider DO report; the OPA +// access control does not, so on a cluster with OPA the auth-revision check +// must be absent from the receipt and the silent component named. The Gateway +// records the check list verbatim, so claiming the check here would write a +// false acknowledgement into the operator's evidence. +func TestValidationDoesNotClaimAnUnacknowledgedAuthRevision(t *testing.T) { + coordinator := newFakeCoordinator(t) + coordinator.sync["securityRevisions"] = []any{ + map[string]any{"kind": "password-authenticator", "name": "file", "revision": fakePasswordRevision}, + map[string]any{"kind": "group-provider", "name": "file", "revision": fakeGroupRevision}, + // What the OPA access control actually reports today. + map[string]any{ + "kind": "system-access-control", "name": "opa", "revision": nil, + "error": "system-access-control 'opa' does not report the configuration it has loaded", + }, + } + + validation, err := coordinator.validate(t, 2, 42) + if err != nil { + t.Fatalf("validate: %v", err) + } + for _, check := range validation.Checks { + if check == trinoPoolCheckAuthRevision { + t.Fatal("the receipt claimed an auth-revision check no component acknowledged") + } + } + if len(validation.Unacknowledged) != 1 || validation.Unacknowledged[0] != "system-access-control/opa" { + t.Fatalf("unacknowledged = %v, want the OPA access control named", validation.Unacknowledged) + } +} + +// When every component does report, the check is claimed and nothing is left +// unacknowledged. +func TestValidationClaimsTheAuthRevisionWhenEveryComponentReports(t *testing.T) { + coordinator := newFakeCoordinator(t) + validation, err := coordinator.validate(t, 2, 42) + if err != nil { + t.Fatalf("validate: %v", err) + } + claimed := false + for _, check := range validation.Checks { + if check == trinoPoolCheckAuthRevision { + claimed = true + } + } + if !claimed { + t.Fatalf("checks = %v, want the auth revision claimed", validation.Checks) + } + if len(validation.Unacknowledged) != 0 { + t.Fatalf("unacknowledged = %v", validation.Unacknowledged) + } +} + +// The image check must COMPARE something. Claiming it unconditionally wrote a +// false acknowledgement into the Gateway's durable evidence, which is exactly +// what the receipt exists to prevent. +func TestValidationRejectsAnImageThatIsNotTheRelease(t *testing.T) { + coordinator := newFakeCoordinator(t) + other := "registry.example.invalid/trino@sha256:2222222222222222222222222222222222222222222222222222222222222222" + + cases := map[string]trinoPoolObservation{ + "coordinator runs another image": { + ReadyWorkers: 2, DesiredWorkers: 2, CoordinatorReady: true, + CoordinatorImage: other, WorkerImage: fakeCoordinatorImage, + }, + "workers run another image": { + ReadyWorkers: 2, DesiredWorkers: 2, CoordinatorReady: true, + CoordinatorImage: fakeCoordinatorImage, WorkerImage: other, + }, + "no image observed at all": { + ReadyWorkers: 2, DesiredWorkers: 2, CoordinatorReady: true, + }, + } + for name, observed := range cases { + t.Run(name, func(t *testing.T) { + _, err := coordinator.validateWith(t, observed, + trinoPoolExpectation{Image: fakeCoordinatorImage, CatalogRevision: 42}) + if !errors.Is(err, errTrinoPoolCandidateNotReady) { + t.Fatalf("error = %v, want the image mismatch to be rejected", err) + } + }) + } +} + +// A candidate at an older catalog revision is structurally healthy and still +// not certified: it would serve a catalog set that does not include the newest +// tenant. The required revision comes from the pool's durable publication +// revision, so this is the check that binds them. +func TestValidationRequiresThePublishedCatalogRevision(t *testing.T) { + coordinator := newFakeCoordinator(t) + coordinator.sync["appliedRevision"] = 41 + + observed := trinoPoolObservation{ + ReadyWorkers: 2, DesiredWorkers: 2, CoordinatorReady: true, + CoordinatorImage: fakeCoordinatorImage, WorkerImage: fakeCoordinatorImage, + } + if _, err := coordinator.validateWith(t, observed, + trinoPoolExpectation{Image: fakeCoordinatorImage, CatalogRevision: 42}); !errors.Is(err, errTrinoPoolCandidateNotReady) { + t.Fatalf("error = %v, want an older applied revision to be refused", err) + } + // At the published revision it passes. + if _, err := coordinator.validateWith(t, observed, + trinoPoolExpectation{Image: fakeCoordinatorImage, CatalogRevision: 41}); err != nil { + t.Fatalf("validate at the published revision: %v", err) + } +} + +// Internal HTTP carries the credential, so the probe must declare the Gateway's +// terminated TLS rather than silently dropping the requirement. A coordinator +// with process-forwarded=true refuses an authenticated request without it. +func TestInternalHTTPProbeDeclaresForwardedHTTPS(t *testing.T) { + var forwarded []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + forwarded = append(forwarded, r.Header.Get("X-Forwarded-Proto")) + if _, _, ok := r.BasicAuth(); !ok { + t.Error("the probe dropped its credential") + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "nodeId": "node-1", "processId": "process-1", "coordinatorId": "abcde", + "enabled": true, "ready": true, "observedRevision": 1, "appliedRevision": 1, + }) + })) + defer server.Close() + + processID, err := probeProcessIdentity(context.Background(), server.Client(), server.URL, + func() (string, string) { return "observer", "secret" }, true) + if err != nil { + t.Fatalf("probe: %v", err) + } + if processID != "process-1" { + t.Fatalf("processId = %q", processID) + } + if len(forwarded) == 0 || forwarded[0] != "https" { + t.Fatalf("forwarded proto = %v, want https declared", forwarded) + } +} + +// A coordinator whose policy engine reports a DIFFERENT projection than the one +// this control plane serves is not certified, however healthy it looks. This is +// the case the check exists for: OPA answers every question perfectly while +// deciding with a bundle that predates the newest tenant, so "the component +// reported something" is not evidence of anything. +func TestValidationDoesNotClaimAStalePolicyRevision(t *testing.T) { + coordinator := newFakeCoordinator(t) + coordinator.sync["securityRevisions"] = []any{ + map[string]any{"kind": "password-authenticator", "name": "file", "revision": fakePasswordRevision}, + map[string]any{"kind": "group-provider", "name": "file", "revision": fakeGroupRevision}, + map[string]any{"kind": "system-access-control", "name": "opa", "revision": "v2.an-older-projection"}, + } + + validation, err := coordinator.validate(t, 2, 42) + if err != nil { + t.Fatalf("validate: %v", err) + } + for _, check := range validation.Checks { + if check == trinoPoolCheckAuthRevision { + t.Fatal("a coordinator deciding with an older projection claimed the auth-revision check") + } + } +} + +// Before this control plane has served a projection it cannot say what a +// coordinator ought to be deciding with, so it claims nothing - and the Gateway +// refuses the admission, which is the fail-closed direction. +func TestValidationDoesNotClaimAnUnknownPolicyRevision(t *testing.T) { + coordinator := newFakeCoordinator(t) + validation, err := coordinator.validateWith(t, trinoPoolObservation{ + ReadyWorkers: 2, DesiredWorkers: 2, CoordinatorReady: true, CoordinatorPodUID: "pod-uid", + CoordinatorImage: fakeCoordinatorImage, WorkerImage: fakeCoordinatorImage, + }, trinoPoolExpectation{Image: fakeCoordinatorImage, CatalogRevision: 42}) + if err != nil { + t.Fatalf("validate: %v", err) + } + for _, check := range validation.Checks { + if check == trinoPoolCheckAuthRevision { + t.Fatal("the auth-revision check was claimed with no served projection to compare against") + } + } +} + +// The OPA bundle and the authentication Secret reach a coordinator by different +// paths and settle at different times. A candidate whose authorization data is +// current but whose password file predates the tenant about to be admitted +// looks healthy and then rejects that tenant's very first request, so each +// projected component is compared on its own. +func TestValidationRequiresEveryProjectedComponentToBeCurrent(t *testing.T) { + for _, testCase := range []struct { + name string + revisions []any + }{ + { + name: "current policy, stale password file", + revisions: []any{ + map[string]any{"kind": "password-authenticator", "name": "file", "revision": "sha256:older"}, + map[string]any{"kind": "group-provider", "name": "file", "revision": fakeGroupRevision}, + map[string]any{"kind": "system-access-control", "name": "opa", "revision": fakePolicyRevision}, + }, + }, + { + name: "current password file, stale groups", + revisions: []any{ + map[string]any{"kind": "password-authenticator", "name": "file", "revision": fakePasswordRevision}, + map[string]any{"kind": "group-provider", "name": "file", "revision": "sha256:older"}, + map[string]any{"kind": "system-access-control", "name": "opa", "revision": fakePolicyRevision}, + }, + }, + { + name: "no password authenticator at all", + revisions: []any{ + map[string]any{"kind": "group-provider", "name": "file", "revision": fakeGroupRevision}, + map[string]any{"kind": "system-access-control", "name": "opa", "revision": fakePolicyRevision}, + }, + }, + { + // A second authenticator reads a file this control plane does not + // write, so it can authenticate principals outside the projection. + // Admitting on the strength of the one component that agrees would + // pull that file inside the pool's trust boundary silently. + name: "a second password authenticator this control plane does not write", + revisions: []any{ + map[string]any{"kind": "password-authenticator", "name": "file", "revision": fakePasswordRevision}, + map[string]any{"kind": "password-authenticator", "name": "file-2", "revision": "sha256:someone-elses-file"}, + map[string]any{"kind": "group-provider", "name": "file", "revision": fakeGroupRevision}, + map[string]any{"kind": "system-access-control", "name": "opa", "revision": fakePolicyRevision}, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + coordinator := newFakeCoordinator(t) + coordinator.sync["securityRevisions"] = testCase.revisions + + validation, err := coordinator.validate(t, 2, 42) + if err != nil { + t.Fatalf("validate: %v", err) + } + for _, check := range validation.Checks { + if check == trinoPoolCheckAuthRevision { + t.Fatal("the auth-revision check was claimed while a projected component was not current") + } + } + }) + } +} + +// With every projected component reporting exactly what is being served, the +// check is claimed. +func TestValidationClaimsTheAuthRevisionWhenEveryProjectionMatches(t *testing.T) { + coordinator := newFakeCoordinator(t) + coordinator.sync["securityRevisions"] = []any{ + map[string]any{"kind": "password-authenticator", "name": "file", "revision": fakePasswordRevision}, + map[string]any{"kind": "group-provider", "name": "file", "revision": fakeGroupRevision}, + map[string]any{"kind": "system-access-control", "name": "opa", "revision": fakePolicyRevision}, + } + + validation, err := coordinator.validate(t, 2, 42) + if err != nil { + t.Fatalf("validate: %v", err) + } + claimed := false + for _, check := range validation.Checks { + if check == trinoPoolCheckAuthRevision { + claimed = true + } + } + if !claimed { + t.Fatalf("checks = %v, want the auth revision claimed", validation.Checks) + } +} diff --git a/controlplane/trino_pool_watermark_test.go b/controlplane/trino_pool_watermark_test.go new file mode 100644 index 000000000..d28f496a6 --- /dev/null +++ b/controlplane/trino_pool_watermark_test.go @@ -0,0 +1,133 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// A catalog that committed while its revision checkpoint failed must not let a +// tenant be admitted against the older revision. +// +// The failure is specific and silent: the catalog publish commits, the write of +// its revision onto the pool row fails, and NOTHING republishes it - the +// catalog already exists, so no later mutation carries the number forward. The +// admission gate then certifies members against a revision that predates this +// tenant, which is how a warehouse is admitted, and reported ready, without its +// catalog. The tenant loop reads the enabled orgs independently of the +// provisioner's outcome, so it never sees that anything failed. +func TestAdmissionStaysClosedUntilTheCatalogWatermarkIsKnown(t *testing.T) { + t.Run("an unreadable watermark admits nothing", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.servingPool(t) + // The tenant arrives while the catalog store cannot be asked what it + // has published. + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.catalogWatermark = func(context.Context) (int64, error) { + return 0, errors.New("catalog store is unreachable") + } + + harness.tickTolerant(10) + + if admitted := harness.gateway.admitted["org-a"]; admitted != "" { + t.Fatalf("a tenant was admitted at %q while the published catalog revision was unknown", admitted) + } + if row := harness.publications.rows["org-a"]; row != nil && row.State == configstore.TrinoPublicationAdmitted { + t.Fatal("a tenant was recorded admitted against an unconfirmed catalog revision") + } + }) + + t.Run("a committed-but-unrecorded revision is recovered", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.servingPool(t) + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + // The store holds a catalog at revision 7; the pool row never got it. + harness.operator.catalogWatermark = func(context.Context) (int64, error) { return 7, nil } + harness.store.pool.PublicationRevision = 3 + + harness.tick(t, 10) + + if harness.store.pool.PublicationRevision != 7 { + t.Fatalf("publication revision = %d, want the store's 7 to be checkpointed", + harness.store.pool.PublicationRevision) + } + if harness.gateway.admitted["org-a"] == "" { + t.Fatalf("the tenant was never admitted after recovery; calls: %v", harness.gateway.calls) + } + }) + + t.Run("a watermark that cannot be checkpointed admits nothing", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.servingPool(t) + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.catalogWatermark = func(context.Context) (int64, error) { return 9, nil } + harness.store.failRevisionCheckpoint = true + + harness.tickTolerant(10) + + if admitted := harness.gateway.admitted["org-a"]; admitted != "" { + t.Fatalf("a tenant was admitted at %q while the watermark could not be checkpointed", admitted) + } + }) + + // Holding admissions must not hold the compute lifecycle: a pool still has + // to repair and drain while its catalog watermark is in doubt. + t.Run("the instance lifecycle keeps running", func(t *testing.T) { + harness := newOperatorHarness(t) + harness.operator.config.Pool.TenantAdmission = true + harness.operator.tenants = &fakeTenantStore{orgs: []configstore.TrinoEnabledOrg{poolOrg("analyst")}} + harness.operator.catalogWatermark = func(context.Context) (int64, error) { + return 0, errors.New("catalog store is unreachable") + } + + harness.tickTolerant(20) + + serving := 0 + for _, instance := range harness.store.instances { + if instance.Phase == string(trinopool.PhaseServing) { + serving++ + } + } + if serving == 0 { + t.Fatalf("no instance reached serving while admissions were held: %v", harness.phases()) + } + if len(harness.gateway.calls) == 0 { + t.Fatal("the instance lifecycle made no progress at all") + } + }) +} + +// A checkpoint failure on the publish path is REPORTED, not logged and dropped: +// the catalog is committed, nothing will republish it, and a silent failure +// leaves the gate certifying members against a revision that predates the +// tenant. +func TestPublishReportsAFailedRevisionCheckpoint(t *testing.T) { + lease := configstore.TrinoPoolLease{PoolID: "registered:cell-001", Owner: "cp-test", Epoch: 4} + writer := &trinoPoolCatalogWriter{ + cellID: "registered:cell-001", + store: refusingRevisionStore{}, + authority: func() (configstore.TrinoPoolLease, bool) { return lease, true }, + } + // No database handle: the publish path fails before the store write, which + // is enough to pin that the checkpoint error is not swallowed - the record + // call itself is exercised through the operator recovery test above. + err := writer.checkpoint(context.Background(), 12) + if err == nil || !strings.Contains(err.Error(), "checkpoint published catalog revision 12") { + t.Fatalf("checkpoint error = %v, want the failure to be surfaced", err) + } +} + +type refusingRevisionStore struct{} + +func (refusingRevisionStore) RecordTrinoPoolPublicationRevision(context.Context, configstore.TrinoPoolLease, string, int64) error { + return errors.New("refused") +} diff --git a/controlplane/trino_pool_wiring.go b/controlplane/trino_pool_wiring.go new file mode 100644 index 000000000..b6573a735 --- /dev/null +++ b/controlplane/trino_pool_wiring.go @@ -0,0 +1,363 @@ +//go:build kubernetes + +package controlplane + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "log/slog" + "os" + "strings" + "sync/atomic" + + "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinogateway" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// Startup wiring for the shared Trino pool. +// +// The operator is constructed at startup and attached to the EXISTING janitor +// leader lease, exactly like the reshard reconciler and the usage collectors. +// With the feature disabled this returns no operators and nothing changes; with +// the pool enabled but the operator disabled, the loops run in a read-only mode +// that keeps the durable desired state in sync and touches nothing external. +// +// Wiring failures are fatal, for the same reason the existing Trino branch is +// fatal: silently skipping would leave an operator believing a pool is being +// reconciled while nothing is. + +// buildTrinoPoolOperators constructs one operator per shared-pool cell. +func buildTrinoPoolOperators( + store *configstore.ConfigStore, + fleet trinoFleet, + controlPlaneID string, +) ([]*trinoPoolOperator, error) { + configs, err := resolveTrinoPoolConfigs() + if err != nil { + return nil, err + } + if len(configs) == 0 { + return nil, nil + } + operatorEnabled := trinoPoolOperatorEnabled() + gateway, err := buildTrinoPoolGateway() + if err != nil { + return nil, err + } + if operatorEnabled && gateway == nil { + // Without a Gateway the operator could create compute but never admit, + // drain or retire it. Creating unadmittable instances is worse than + // refusing to start. + return nil, fmt.Errorf("%s is enabled but %s is not configured", envTrinoPoolOperatorEnabled, envTrinoPoolGatewayURL) + } + + // The owner identity must be unique per PROCESS, not merely per control + // plane: two processes sharing an identity would both satisfy the fence's + // owner check and the epoch would stop distinguishing them. + owner := trinoPoolOwnerIdentity(controlPlaneID) + + operators := make([]*trinoPoolOperator, 0, len(configs)) + for _, config := range configs { + // Credentials and the Kubernetes client are taken from THIS pool's own + // cell. Picking the first entry of a map meant a pool could be certified + // with another cell's observer credential, chosen by Go's randomized map + // order - a different answer on different boots. + wire := fleet.byStoredID(config.PoolID) + if wire == nil { + return nil, fmt.Errorf("shared Trino pool %s has no wired cell", config.PublicID) + } + if wire.Kubernetes == nil { + return nil, fmt.Errorf("shared Trino pool %s has no Kubernetes client", config.PublicID) + } + if wire.Provisioner == nil { + return nil, fmt.Errorf("shared Trino pool %s has no operational credential", config.PublicID) + } + clientset := wire.Kubernetes + observerCredential := wire.Provisioner.ObserverCredential + + operator := &trinoPoolOperator{ + config: config, + store: store, + gateway: gateway, + owner: owner, + operatorEnabled: operatorEnabled, + newInstanceID: newTrinoPoolInstanceID, + tenants: store, + publications: store, + operations: store, + } + // Desired state is published from the ConfigMap the chart projects the + // registry and blueprint from, read through the Kubernetes API. + // + // The mounted copies are how this process learned the pool exists, and + // that is all they are trusted for. A projected volume is refreshed per + // pod on the kubelet's own schedule - and not at all under a subPath + // mount - so two replicas can hold different contents indefinitely and + // an idle pod can hold a configuration the cluster replaced hours ago. + // Publishing from the API object means every replica derives desired + // state from one value, and re-reading it before each publication is + // what bounds the staleness window to a single read. + configSource, err := newTrinoPoolAPIConfigReader(clientset, config.Namespace) + if err != nil { + return nil, fmt.Errorf("configure the desired-state source for pool %s: %w", config.PublicID, err) + } + publicID := config.PublicID + operator.resolveConfig = func() (trinoPoolConfig, error) { + return resolveTrinoPoolConfigByID(context.Background(), configSource, publicID) + } + // The Kubernetes effects follow the CURRENT configuration, because a new + // blueprint may name different pool-shared resources, and those are the + // objects the effects refuse to touch. + operator.kube = func(epoch int64) trinoPoolKube { + shared := trinopool.BlueprintSharedResources{} + if operator.config.Blueprint != nil { + shared = operator.config.Blueprint.SharedResources + } + return newTrinoPoolEffects(clientset, operator.config.Namespace, shared, epoch) + } + // Pooled coordinators are reached directly on their in-cluster Service + // over plain HTTP: TLS terminates at the Gateway. The probe declares the + // forwarded HTTPS hop rather than relaxing authentication, so a + // coordinator that rejects credentials over unforwarded HTTP keeps + // rejecting them. + client := newRolloutHTTPClient("") + operator.validate = func(ctx context.Context, endpoint string, observed trinoPoolObservation, expected trinoPoolExpectation) (trinoPoolValidation, error) { + return validateTrinoPoolCandidate(ctx, client, endpoint, observerCredential, observed, expected) + } + operator.identity = func(ctx context.Context, endpoint string) (string, error) { + return probeProcessIdentity(ctx, client, endpoint, observerCredential, true) + } + // What a publication receipt asserts about ONE member, read from that + // member rather than assumed from what was published. + operator.acknowledgement = func(ctx context.Context, endpoint string, expected trinoPoolProjectionRevisions, catalogRevision int64) (trinoPoolAcknowledgement, error) { + return probeMemberAcknowledgement(ctx, client, endpoint, observerCredential, expected, catalogRevision) + } + // The projection is produced by THIS cell's provisioner: it serves the + // bundle the candidate's OPA pulls and writes the Secret the candidate + // mounts its password and group files from. + provisionerForProjection := wire.Provisioner + operator.projection = func() trinoPoolProjectionRevisions { + password, group := provisionerForProjection.PublishedAuthRevisions() + return trinoPoolProjectionRevisions{ + Policy: provisionerForProjection.PublishedPolicyRevision(), + Password: password, + Group: group, + } + } + + // The lease this process currently holds over the pool. + // + // It is an atomic pointer because the two sides run on different + // goroutines: the operator's leader loop writes it, and the + // provisioner's per-cell reconcile goroutines read it on every catalog + // publication and every projection. A plain captured variable was a + // data race, and the value it raced on decides whether a write is + // fenced at all. + authority := &atomic.Pointer[configstore.TrinoPoolLease]{} + + // The projection fence. A pooled cell's authorization and + // authentication projections are accepted by the control plane's own + // durable record, and only by a process running the image the + // deployment currently wants: an older binary that won the lease would + // otherwise publish its own older rules under a newer revision, which a + // counter cannot detect. + // + // Both inputs are rendered by the chart from one image helper: this + // process's own startup environment, and the pool ConfigMap, which is + // re-read immediately before every publication and carries the desired + // publisher image alongside the desired configuration. + producer := newTrinoPoolProducerIdentity() + // Serving is fenced by the same record, on EVERY replica - not just the + // one holding the authority. A replica whose projection has been + // replaced must stop handing it to coordinators, and it is precisely + // the replica that does not know it is behind. + accepted := &trinoPoolAcceptedProjection{store: store, poolID: config.PoolID} + if wire.BundleHandler != nil { + wire.BundleHandler.AcceptedRevision = accepted.digest + } + // Candidate admission compares against the same durable record, for the + // same reason: what THIS process published is not evidence about what + // the pool accepts. + operator.acceptedProjection = func() string { + digest, known := accepted.digest() + if !known { + return "" + } + return digest + } + provisionerForProjection.SetProjectionFence(&trinoPoolProjectionFence{ + store: store, + publicID: config.PublicID, + authority: authority, + configSource: configSource, + producer: producer, + accepted: accepted, + }) + + // With the Gateway's admission restriction on, a warehouse is not + // queryable until its publication barrier commits - so it must not read + // as Ready before then. The durable publication record is the answer; + // the Gateway's own state is authoritative for it and is what wrote it. + if config.Pool.TenantAdmission { + poolID := config.PoolID + wire.Provisioner.SetTenantAdmissionGate(func(orgID string) (bool, string) { + publication, err := store.GetTrinoPoolPublication(context.Background(), poolID, orgID) + if err != nil { + return false, "the pool's publication state is unreadable" + } + return trinoPoolTenantIsAdmitted(publication) + }) + } + + // The fenced catalog writer, if this deployment has moved the cell off + // the coordinator-mediated path. Its fence is the pool authority, so it + // is claimed and installed when the operator wins the lease - never at + // startup, where every replica would claim it. + // + // It publishes under the SAME lease the projection fence uses. + writer, err := buildTrinoPoolCatalogWriter(config.PoolID, store, + func() (configstore.TrinoPoolLease, bool) { + lease := authority.Load() + if lease == nil { + return configstore.TrinoPoolLease{}, false + } + return *lease, true + }, + // Node inventory still comes from a live coordinator: the catalog + // store knows nothing about cluster membership. For a pooled cell + // that is whichever instance the operator is currently validating, + // so the bridge is given no static node client and the provisioner's + // readiness check reads the pool's instances instead. + nil) + if err != nil { + return nil, fmt.Errorf("configure catalog writer for pool %s: %w", config.PublicID, err) + } + if writer != nil { + provisionerForCell := wire.Provisioner + // The admission gate's revision is read from the catalog store + // itself, so a checkpoint that failed after a committed catalog is + // recovered rather than waiting for a mutation that will never come. + operator.catalogWatermark = writer.PublishedRevision + operator.installWriter = func(ctx context.Context, lease configstore.TrinoPoolLease) error { + held := lease + authority.Store(&held) + if err := writer.ClaimWriter(ctx); err != nil { + authority.Store(nil) + return err + } + provisionerForCell.SetCatalogClient(writer) + slog.Info("Shared Trino pool catalog writer claimed.", + "pool", config.PublicID, "epoch", lease.Epoch) + return nil + } + // When the leadership term ends - cancelled or fenced - the writer + // stops being able to publish. Leaving the lease behind would let a + // superseded process keep issuing writes that are only refused at + // the store's own fence, one failed catalog publication at a time. + operator.releaseWriter = func() { authority.Store(nil) } + } + operators = append(operators, operator) + + slog.Info("Shared Trino pool configured.", + "pool", config.PublicID, "namespace", config.Namespace, + "desired", config.Spec.DesiredInstances, "minServing", config.Spec.MinServing, + "operator", operatorEnabled, "frozen", config.Frozen, "reason", config.FrozenReason) + } + return operators, nil +} + +// buildTrinoPoolGateway builds the pooled-lifecycle client from the existing +// managed-gateway configuration. It returns (nil, nil) when no Gateway is +// configured, which is valid while the operator is disabled. +func buildTrinoPoolGateway() (trinoPoolGateway, error) { + endpoint := strings.TrimSpace(os.Getenv(envTrinoPoolGatewayURL)) + if endpoint == "" { + endpoint = strings.TrimSpace(os.Getenv("DUCKGRES_TRINO_MANAGED_GATEWAY_URL")) + } + if endpoint == "" { + return nil, nil + } + token, err := readTrinoPoolGatewayToken() + if err != nil { + return nil, err + } + client, err := trinogateway.NewClient(trinogateway.Config{ + BaseURL: endpoint, + AdminToken: token, + TLSServerName: strings.TrimSpace(os.Getenv("DUCKGRES_TRINO_MANAGED_GATEWAY_SERVER_NAME")), + }) + if err != nil { + return nil, fmt.Errorf("configure shared-pool Gateway client: %w", err) + } + return client, nil +} + +// readTrinoPoolGatewayToken reuses the existing rollout capability token file. +// The pooled protocol authenticates with the Gateway's existing admin +// credential and introduces no new secret. +func readTrinoPoolGatewayToken() (string, error) { + path := strings.TrimSpace(os.Getenv("DUCKGRES_TRINO_ROLLOUT_TOKEN_FILE")) + if path == "" { + return "", fmt.Errorf("a shared-pool Gateway is configured but DUCKGRES_TRINO_ROLLOUT_TOKEN_FILE is unset") + } + token, err := readRolloutSecretFile(path, 4096) + if err != nil { + return "", err + } + return strings.TrimSpace(string(token)), nil +} + +// attachTrinoPoolOperators registers the reconcile loops under the janitor +// leader lease. One control plane reconciles a pool at a time; the authority +// epoch is what makes that safe rather than merely likely. +func attachTrinoPoolOperators(leader *JanitorLeaderManager, operators []*trinoPoolOperator) { + if leader == nil { + return + } + for _, operator := range operators { + leader.AttachLeaderLoop(operator.Run) + } +} + +// trinoPoolOwnerIdentity makes the fence owner unique per process. The control +// plane instance id survives a restart, so two processes could otherwise both +// match the recorded owner and the epoch alone would decide - which is exactly +// the ambiguity the owner check exists to remove. +func trinoPoolOwnerIdentity(controlPlaneID string) string { + buffer := make([]byte, 8) + if _, err := rand.Read(buffer); err != nil { + // Fall back to the pid: still per-process on this node, and better than + // a shared constant. + return fmt.Sprintf("%s.pid-%d", controlPlaneID, os.Getpid()) + } + return controlPlaneID + "." + hex.EncodeToString(buffer) +} + +// trinoPoolTenantIsAdmitted answers the Ready gate: may this warehouse be +// reported as queryable? +// +// The question is whether it has EVER been admitted and still is - not whether +// a barrier happens to be open right now. Adding a login opens a new attempt, +// and the publication's state leaves `admitted` while that attempt runs, so a +// state-only reading would flap a warehouse that has been serving for weeks +// back to Provisioning because somebody created a user. The committed target +// revision is the durable evidence that the Gateway has dispatched for it. +// +// A revoked tenant is not serving, one that never committed a barrier was never +// dispatchable, and a missing record is not evidence of anything - all three +// wait. +func trinoPoolTenantIsAdmitted(publication *configstore.TrinoPoolPublication) (bool, string) { + if publication == nil { + return false, "waiting for the pool to publish this warehouse" + } + if publication.State == configstore.TrinoPublicationRevoked { + return false, "this warehouse's admission was revoked" + } + if publication.AdmittedTargetRevision == "" { + return false, "waiting for the pool to admit this warehouse: " + publication.State + } + return true, "" +} diff --git a/controlplane/trino_pool_wiring_test.go b/controlplane/trino_pool_wiring_test.go new file mode 100644 index 000000000..d35adef2d --- /dev/null +++ b/controlplane/trino_pool_wiring_test.go @@ -0,0 +1,193 @@ +//go:build kubernetes + +package controlplane + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/posthog/duckgres/controlplane/configstore" + + "github.com/posthog/duckgres/controlplane/provisioner" + "k8s.io/client-go/kubernetes/fake" +) + +// testPoolFleet wires one cell whose stored id matches the pooled registry +// fixture, so the operator can find ITS OWN credentials rather than whichever +// map entry came first. +func testPoolFleet() trinoFleet { + return trinoFleet{{ + Cell: trinoCell{ID: registeredTrinoCellPrefix + "cell-001", PublicID: "cell-001", Mode: trinoPoolModeShared}, + Kubernetes: fake.NewClientset(), + Provisioner: &provisioner.TrinoProvisioner{}, + }} +} + +// With no pooled cell in the registry the wiring produces nothing. This is what +// "ships disabled" has to mean at the startup boundary: the code path is +// constructed on every boot, and on a fleet that has not opted in it resolves +// to zero operators and zero side effects. +func TestPoolWiringIsInertWithoutAPooledCell(t *testing.T) { + t.Setenv(envTrinoCellsFile, "") + t.Setenv(envTrinoPoolEnabled, "") + t.Setenv(envTrinoPoolOperatorEnabled, "") + + operators, err := buildTrinoPoolOperators(nil, nil, "cp-test") + if err != nil { + t.Fatalf("wiring failed on a fleet without pools: %v", err) + } + if len(operators) != 0 { + t.Fatalf("built %d operators", len(operators)) + } +} + +// A configured pool with the operator enabled but no Gateway would create +// compute it can never admit, drain or retire. Refusing to start is the +// smaller failure. +func TestPoolWiringRefusesAnOperatorWithoutAGateway(t *testing.T) { + blueprint := filepath.Join(t.TempDir(), "blueprint.json") + if err := os.WriteFile(blueprint, testBlueprintJSON(t), 0o600); err != nil { + t.Fatalf("write blueprint: %v", err) + } + t.Setenv(envTrinoCellsFile, sharedPoolRegistry(t, blueprint)) + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + t.Setenv(envTrinoPoolOperatorEnabled, "true") + t.Setenv(envTrinoPoolConfigMap, "duckgres-trino-pool") + t.Setenv(envTrinoPoolGatewayURL, "") + t.Setenv("DUCKGRES_TRINO_MANAGED_GATEWAY_URL", "") + + _, err := buildTrinoPoolOperators(nil, testPoolFleet(), "cp-test") + if err == nil { + t.Fatal("an enabled operator without a Gateway was accepted") + } +} + +// A pooled cell with the operator disabled still wires: the durable desired +// state is kept in sync and nothing external is touched. +func TestPoolWiringBuildsAReadOnlyOperator(t *testing.T) { + blueprint := filepath.Join(t.TempDir(), "blueprint.json") + if err := os.WriteFile(blueprint, testBlueprintJSON(t), 0o600); err != nil { + t.Fatalf("write blueprint: %v", err) + } + t.Setenv(envTrinoCellsFile, sharedPoolRegistry(t, blueprint)) + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + t.Setenv(envTrinoPoolOperatorEnabled, "false") + // Desired state is published from the API object, so the pool has to be + // told which one. + t.Setenv(envTrinoPoolConfigMap, "duckgres-trino-pool") + t.Setenv(envTrinoPoolGatewayURL, "") + t.Setenv("DUCKGRES_TRINO_MANAGED_GATEWAY_URL", "") + + operators, err := buildTrinoPoolOperators(nil, testPoolFleet(), "cp-test") + if err != nil { + t.Fatalf("wiring failed: %v", err) + } + if len(operators) != 1 { + t.Fatalf("built %d operators", len(operators)) + } + if operators[0].operatorEnabled { + t.Fatal("the operator was built enabled") + } + // The owner is the control-plane id plus a per-PROCESS suffix: two + // processes of the same control plane must not both satisfy the fence's + // owner check, or the epoch is the only thing telling them apart. + if !strings.HasPrefix(operators[0].owner, "cp-test.") || operators[0].owner == "cp-test." { + t.Fatalf("owner = %q, want a per-process identity under cp-test", operators[0].owner) + } + second, err := buildTrinoPoolOperators(nil, testPoolFleet(), "cp-test") + if err != nil { + t.Fatalf("second wiring: %v", err) + } + if second[0].owner == operators[0].owner { + t.Fatal("two processes of the same control plane received the same owner identity") + } +} + +// A pooled cell whose desired-state source is not named refuses to start. +// +// The alternative would be publishing desired state from this pod's mounted +// copy of the configuration, which lags per pod and never updates at all under +// a subPath mount - so two replicas could drive one pool from two different +// configurations, indefinitely, with nothing saying which. A refusal at startup +// is the smaller failure, and it is the same rule the rest of the Trino branch +// follows: asking for the feature and getting a silently different shape is +// worse than not starting. +func TestPoolWiringRefusesWithoutADesiredStateSource(t *testing.T) { + blueprint := filepath.Join(t.TempDir(), "blueprint.json") + if err := os.WriteFile(blueprint, testBlueprintJSON(t), 0o600); err != nil { + t.Fatalf("write blueprint: %v", err) + } + t.Setenv(envTrinoCellsFile, sharedPoolRegistry(t, blueprint)) + t.Setenv(envTrinoRegistryOnly, "true") + t.Setenv(envTrinoPoolEnabled, "true") + t.Setenv(envTrinoPoolOperatorEnabled, "false") + t.Setenv(envTrinoPoolConfigMap, "") + + if _, err := buildTrinoPoolOperators(nil, testPoolFleet(), "cp-test"); err == nil { + t.Fatal("a pooled cell was wired with no authoritative desired-state source") + } +} + +// The Ready gate asks whether this warehouse has ever been admitted and still +// is - not whether a barrier happens to be open right now. +// +// Adding a login opens a new attempt, and the publication's state leaves +// `admitted` while that attempt runs. Reading the state alone would flap a +// warehouse that has been serving for weeks back to Provisioning because +// somebody created a user. +func TestReadyGateFollowsAdmissionNotTheAttemptInFlight(t *testing.T) { + for _, testCase := range []struct { + name string + publication configstore.TrinoPoolPublication + ready bool + }{ + { + name: "serving while a new attempt is in flight", + publication: configstore.TrinoPoolPublication{ + State: configstore.TrinoPublicationAdmitting, + AdmittedTargetRevision: "b0123456789a.a1", + }, + ready: true, + }, + { + name: "admitted and idle", + publication: configstore.TrinoPoolPublication{ + State: configstore.TrinoPublicationAdmitted, + AdmittedTargetRevision: "b0123456789a.a1", + }, + ready: true, + }, + { + name: "never admitted", + publication: configstore.TrinoPoolPublication{State: configstore.TrinoPublicationPublished}, + ready: false, + }, + { + name: "revoked", + publication: configstore.TrinoPoolPublication{ + State: configstore.TrinoPublicationRevoked, + AdmittedTargetRevision: "", + }, + ready: false, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + ready, reason := trinoPoolTenantIsAdmitted(&testCase.publication) + if ready != testCase.ready { + t.Fatalf("ready = %v (%q), want %v", ready, reason, testCase.ready) + } + if !ready && reason == "" { + t.Fatal("a warehouse held at Provisioning must say why") + } + }) + } + + // An unreadable record is not a reason to report a warehouse ready. + if ready, reason := trinoPoolTenantIsAdmitted(nil); ready || reason == "" { + t.Fatalf("a missing publication reported ready=%v (%q)", ready, reason) + } +} diff --git a/controlplane/trino_registry.go b/controlplane/trino_registry.go index c934fa987..bb0da957c 100644 --- a/controlplane/trino_registry.go +++ b/controlplane/trino_registry.go @@ -81,7 +81,7 @@ func resolveTrinoCells() ([]trinoCell, error) { if !registryOnly && entry.Namespace == legacyNS { return nil, errors.New("registered cell must not share the legacy namespace") } - cell := trinoCell{ID: registeredTrinoCellPrefix + entry.ID, PublicID: entry.ID, RoutingGroup: entry.RoutingGroup, Namespace: entry.Namespace, ClientURL: entry.ClientURL, Backends: entry.Backends, CatalogManagement: entry.CatalogManagement} + cell := trinoCell{Mode: strings.TrimSpace(entry.Mode), ID: registeredTrinoCellPrefix + entry.ID, PublicID: entry.ID, RoutingGroup: entry.RoutingGroup, Namespace: entry.Namespace, ClientURL: entry.ClientURL, Backends: entry.Backends, CatalogManagement: entry.CatalogManagement} for _, backend := range entry.Backends { endpoint, _ := trinoEndpointKey(backend.CoordinatorURL) if !registryOnly && endpoint == legacyEndpoint { @@ -97,12 +97,18 @@ func resolveTrinoCells() ([]trinoCell, error) { } type trinoRegisteredCell struct { - CatalogManagement string `json:"catalog_management,omitempty"` - ID string `json:"id"` - Namespace string `json:"namespace"` - ClientURL string `json:"client_url"` - RoutingGroup string `json:"routing_group"` - Backends []trinoRegisteredBackend `json:"backends"` + CatalogManagement string `json:"catalog_management,omitempty"` + // Mode selects the compute topology: empty or "fixed" is today's blue/green + // cell, "shared-pool" is the operator-managed pool. Unknown values are + // rejected rather than defaulted, so a newer config file cannot be + // half-understood by an older binary. + Mode string `json:"mode,omitempty"` + Pool *trinoRegisteredPool `json:"pool,omitempty"` + ID string `json:"id"` + Namespace string `json:"namespace"` + ClientURL string `json:"client_url"` + RoutingGroup string `json:"routing_group"` + Backends []trinoRegisteredBackend `json:"backends"` } type trinoRegisteredBackend struct { @@ -140,6 +146,10 @@ func parseTrinoCellRegistry(data []byte) ([]trinoRegisteredCell, error) { if cell.CatalogManagement != "" && (len(cell.Backends) != 2 || cell.Backends[0].ID == cell.Backends[1].ID || (cell.Backends[0].ID != "blue" && cell.Backends[0].ID != "green") || (cell.Backends[1].ID != "blue" && cell.Backends[1].ID != "green")) { return nil, errors.New("managed Trino cell requires exactly blue and green slots") } + // A shared-pool cell has no static backends: its members are created + // by the operator and recorded durably. The per-backend checks below + // therefore do not apply to it. + pooled := strings.TrimSpace(cell.Mode) == trinoPoolModeShared if cell.ID == "legacy" || len(validation.IsDNS1123Label(cell.ID)) != 0 { return nil, errors.New("Trino cell identity must be a DNS label other than legacy") } @@ -192,7 +202,7 @@ func parseTrinoCellRegistry(data []byte) ([]trinoRegisteredCell, error) { active++ } } - if active != 1 { + if active != 1 && !pooled { return nil, fmt.Errorf("Trino cell %s must have exactly one routing-active backend", cell.ID) } } diff --git a/controlplane/trino_rollout_probe.go b/controlplane/trino_rollout_probe.go index 99407808b..ee30d652f 100644 --- a/controlplane/trino_rollout_probe.go +++ b/controlplane/trino_rollout_probe.go @@ -116,12 +116,33 @@ type rolloutSQLClient struct { baseURL string client *http.Client username, password string + // internalHTTP marks a pooled coordinator reached directly on its + // in-cluster Service over plain HTTP. TLS terminates at the Gateway, and + // the coordinator runs with http-server.process-forwarded=true, so an + // authenticated request has to carry the forwarded-HTTPS metadata the + // Gateway would supply. Authentication is NOT relaxed anywhere: without + // these headers Trino refuses the credential rather than accepting it in + // the clear. The legacy fixed-cell path leaves this false and keeps its + // HTTPS-only checks byte for byte. + internalHTTP bool +} + +// forwardedScheme is what a coordinator behind a TLS-terminating proxy must be +// told about the original request. It is a statement about the Gateway hop, +// not a way to bypass the coordinator's own authentication. +const forwardedScheme = "https" + +func (c rolloutSQLClient) scheme() string { + if c.internalHTTP { + return "http" + } + return "https" } func (c rolloutSQLClient) read(ctx context.Context, method, endpoint, sql string) ([]byte, error) { base, baseErr := url.Parse(c.baseURL) parsed, err := url.Parse(endpoint) - if baseErr != nil || err != nil || parsed.Scheme != "https" || parsed.Scheme != base.Scheme || !strings.EqualFold(parsed.Hostname(), base.Hostname()) || rolloutHTTPSPort(parsed) != rolloutHTTPSPort(base) || parsed.User != nil || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || (parsed.Path != "/v1/statement" && !strings.HasPrefix(parsed.Path, "/v1/statement/") && parsed.Path != "/v1/info") { + if baseErr != nil || err != nil || parsed.Scheme != c.scheme() || parsed.Scheme != base.Scheme || !strings.EqualFold(parsed.Hostname(), base.Hostname()) || rolloutHTTPSPort(parsed) != rolloutHTTPSPort(base) || parsed.User != nil || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" || (parsed.Path != "/v1/statement" && !strings.HasPrefix(parsed.Path, "/v1/statement/") && parsed.Path != "/v1/info") { return nil, errors.New("invalid coordinator response endpoint") } if c.username == "" || c.password == "" { @@ -135,6 +156,14 @@ func (c rolloutSQLClient) read(ctx context.Context, method, endpoint, sql string req.Header.Set("X-Trino-User", c.username) req.Header.Set("X-Trino-Source", "rollout-readiness") req.Header.Set("Content-Type", "text/plain") + if c.internalHTTP { + // Declare the Gateway's terminated TLS. A coordinator with + // process-forwarded=true reads these; without them it rejects an + // authenticated request over plain HTTP, which is the behavior we + // want to keep rather than disable. + req.Header.Set("X-Forwarded-Proto", forwardedScheme) + req.Header.Set("X-Forwarded-Port", "443") + } response, err := c.client.Do(req) if err != nil { return nil, errors.New("coordinator request failed") @@ -154,6 +183,9 @@ func rolloutHTTPSPort(endpoint *url.URL) string { if port := endpoint.Port(); port != "" { return port } + if endpoint.Scheme == "http" { + return "80" + } return "443" } diff --git a/controlplane/trinocatalog/publisher.go b/controlplane/trinocatalog/publisher.go new file mode 100644 index 000000000..2b4e667a5 --- /dev/null +++ b/controlplane/trinocatalog/publisher.go @@ -0,0 +1,546 @@ +// Package trinocatalog is the fenced direct publisher for the shared Trino +// catalog store. +// +// Today a catalog is created by issuing CREATE CATALOG against a coordinator, +// which then writes the shared `trino_catalogs` table itself. That routes +// provisioning through a replaceable compute instance and inherits the +// asynchronous SQL-DDL cancellation ambiguity: a timed-out statement has an +// unknown outcome that cannot be resolved. Here duckgres writes the store +// directly, under a row-lock fence, with a mutation journal that turns a lost +// COMMIT response into a lookup instead of a guess. Serving coordinators run as +// managed readers with read-only credentials. +// +// The physical schema is owned by the Trino-side catalog store. The exact +// table definitions this writer depends on are created by EnsureSchema below +// and asserted against a real PostgreSQL in tests/trinocatalog/. +package trinocatalog + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "regexp" + "sort" + "strings" + "time" + + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// Operations recorded in the journal. The spelling is part of the cross-repo +// contract. +const ( + OperationAddOrReplace = "ADD_OR_REPLACE" + OperationRemove = "REMOVE" +) + +// Budgets. The overall budget bounds one mutation; the lock and statement +// timeouts keep a stuck publisher from holding the cell's writer row. +const ( + DefaultTimeout = 5 * time.Second + lockTimeout = "1s" + statementTimeout = "3s" + + maxCatalogNameLength = 255 + maxConnectorLength = 255 + maxOperationIDLength = 256 + maxProperties = 256 + maxPropertyValueLength = 8192 +) + +var ( + // ErrFenced means a newer writer owns the cell. The mutation was not + // applied and must never be retried under the old epoch. + ErrFenced = errors.New("trino catalog writer is fenced by a newer epoch") + // ErrNotWriter means this process is not the recorded writer. It is + // deliberately distinct from ErrFenced: it also covers the "I believe I am + // newer" case, which must NOT silently seize the cell. + ErrNotWriter = errors.New("trino catalog writer identity or epoch does not match the recorded writer") + // ErrIntentChanged means the same operation id was reused with different + // content. That is a caller bug, never a replay. + ErrIntentChanged = errors.New("trino catalog operation was replayed with different content") + + catalogNamePattern = regexp.MustCompile(`^[a-z0-9_][a-z0-9_-]*$`) + operationIDPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,256}$`) + writerIdentityMatch = regexp.MustCompile(`^[A-Za-z0-9_.:-]{1,255}$`) +) + +// Publisher writes one cell's catalogs. Its epoch and identity are the fence: +// both must match the recorded writer state for a mutation to apply. +type Publisher struct { + db *sql.DB + cellID string + identity string + epoch int64 + timeout time.Duration +} + +// State is the writer-state row as the readers see it. +type State struct { + Revision int64 + WriterEpoch int64 + Identity string + CatalogCount int +} + +// Result is the outcome of one mutation. Replayed marks a journal hit, which is +// how a lost COMMIT response is resolved. +type Result struct { + Revision int64 + CatalogCount int + Replayed bool +} + +// Mutation is one catalog definition change. +type Mutation struct { + OperationID string + Operation string + CatalogName string + ConnectorName string + Properties map[string]string +} + +// NewPublisher validates the writer identity. epoch must be positive: 0 is the +// seeded "nobody has ever written" value and must not be claimable. +func NewPublisher(db *sql.DB, cellID, identity string, epoch int64) (*Publisher, error) { + if db == nil { + return nil, errors.New("catalog publisher requires a database handle") + } + if cellID == "" || len(cellID) > 255 || !writerIdentityMatch.MatchString(cellID) { + return nil, errors.New("catalog publisher requires a valid cell id") + } + if !writerIdentityMatch.MatchString(identity) { + return nil, errors.New("catalog publisher requires a valid writer identity") + } + if epoch < 1 { + return nil, errors.New("catalog publisher requires a positive writer epoch") + } + return &Publisher{db: db, cellID: cellID, identity: identity, epoch: epoch, timeout: DefaultTimeout}, nil +} + +// CellID reports the cell this publisher owns. +func (p *Publisher) CellID() string { return p.cellID } + +// Epoch reports the fence this publisher writes under. +func (p *Publisher) Epoch() int64 { return p.epoch } + +// EnsureSchema creates the additive tables. A managed-reader coordinator runs no +// DDL at all, so somebody has to; this is that owner. It is only called when +// bootstrap is explicitly enabled. +func (p *Publisher) EnsureSchema(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + statements := []string{ + `CREATE TABLE IF NOT EXISTS trino_catalogs ( + cell_id varchar NOT NULL, + catalog_name varchar NOT NULL, + connector_name varchar NOT NULL, + catalog_version varchar NOT NULL, + properties text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (cell_id, catalog_name) + )`, + `CREATE TABLE IF NOT EXISTS trino_catalog_writer_state ( + cell_id varchar NOT NULL, + revision bigint NOT NULL, + writer_epoch bigint NOT NULL, + writer_identity varchar NOT NULL, + catalog_count integer NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (cell_id) + )`, + `CREATE TABLE IF NOT EXISTS trino_catalog_journal ( + cell_id varchar NOT NULL, + revision bigint NOT NULL, + operation_id varchar NOT NULL, + operation varchar NOT NULL, + catalog_name varchar NOT NULL, + catalog_version varchar, + payload_hash varchar NOT NULL, + writer_epoch bigint NOT NULL, + committed_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (cell_id, revision) + )`, + `CREATE UNIQUE INDEX IF NOT EXISTS trino_catalog_journal_operation + ON trino_catalog_journal (cell_id, operation_id)`, + } + for _, statement := range statements { + if _, err := p.db.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("catalog store bootstrap: %w", err) + } + } + return nil +} + +// State seeds the writer-state row if it is missing and returns it. The seed is +// the one place where catalog_count is derived from the existing rows: an +// initialized store that already holds catalogs must never be described as +// holding zero, or every reader's completeness check fails and the fleet +// freezes on last-good state. +func (p *Publisher) State(ctx context.Context) (State, error) { + var state State + err := p.inTransaction(ctx, func(tx *sql.Tx) error { + var err error + state, err = p.lockWriterState(ctx, tx) + return err + }) + return state, err +} + +// Takeover is the explicit, serialized way to become the writer. It is separate +// from Apply on purpose: a mutation must never claim a higher epoch as a side +// effect, or a process that merely believes it is newer could seize the cell in +// the middle of somebody else's write. Taking over locks the same row a +// mutation locks, so an in-flight write either commits before the takeover or +// fails its fence check afterwards. A lease expiry alone decides nothing here. +func (p *Publisher) Takeover(ctx context.Context) (State, error) { + var state State + err := p.inTransaction(ctx, func(tx *sql.Tx) error { + current, err := p.lockWriterState(ctx, tx) + if err != nil { + return err + } + if current.WriterEpoch > p.epoch { + return fmt.Errorf("%w: recorded epoch %d is newer than %d", ErrFenced, current.WriterEpoch, p.epoch) + } + // An EQUAL epoch held by a different identity is not a takeover, it is a + // collision: two writers believe they are the same authority. Letting the + // second one overwrite the identity silently would leave both passing the + // mutation fence, which is exactly the ambiguity the identity check + // exists to remove. Only a strictly higher epoch may claim the cell. + if current.WriterEpoch == p.epoch && current.Identity != "" && current.Identity != p.identity { + return fmt.Errorf("%w: epoch %d is already held by %q", ErrFenced, current.WriterEpoch, current.Identity) + } + if _, err := tx.ExecContext(ctx, + `UPDATE trino_catalog_writer_state SET writer_epoch = $2, writer_identity = $3, updated_at = now() WHERE cell_id = $1`, + p.cellID, p.epoch, p.identity); err != nil { + return fmt.Errorf("claim catalog writer: %w", err) + } + state = State{Revision: current.Revision, WriterEpoch: p.epoch, Identity: p.identity, CatalogCount: current.CatalogCount} + return nil + }) + return state, err +} + +// Apply publishes one catalog mutation. Everything happens in a single +// transaction: fence check, replay resolution, the definition change, the +// revision bump, the recomputed count and the journal row. Either all of it is +// visible to a reader or none of it is. +func (p *Publisher) Apply(ctx context.Context, mutation Mutation) (Result, error) { + if err := mutation.validate(); err != nil { + return Result{}, err + } + var result Result + err := p.inTransaction(ctx, func(tx *sql.Tx) error { + current, err := p.lockWriterState(ctx, tx) + if err != nil { + return err + } + if err := p.checkFence(current); err != nil { + return err + } + + // Replay resolution comes before any effect, so a retried intent can + // never publish twice. + if recorded, found, err := p.journalEntry(ctx, tx, mutation.OperationID); err != nil { + return err + } else if found { + if recorded.payloadHash != mutation.PayloadHash() { + return fmt.Errorf("%w: operation %q", ErrIntentChanged, mutation.OperationID) + } + result = Result{Revision: recorded.revision, CatalogCount: current.CatalogCount, Replayed: true} + return nil + } + + version, err := p.applyDefinition(ctx, tx, mutation) + if err != nil { + return err + } + + revision := current.Revision + 1 + count, err := p.countCatalogs(ctx, tx) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, + `UPDATE trino_catalog_writer_state + SET revision = $2, catalog_count = $3, updated_at = now() + WHERE cell_id = $1`, + p.cellID, revision, count); err != nil { + return fmt.Errorf("advance catalog revision: %w", err) + } + if _, err := tx.ExecContext(ctx, + `INSERT INTO trino_catalog_journal + (cell_id, revision, operation_id, operation, catalog_name, catalog_version, payload_hash, writer_epoch) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8)`, + p.cellID, revision, mutation.OperationID, mutation.Operation, mutation.CatalogName, + version, mutation.PayloadHash(), p.epoch); err != nil { + return fmt.Errorf("record catalog journal: %w", err) + } + result = Result{Revision: revision, CatalogCount: count} + return nil + }) + return result, err +} + +// ResolveOperation reports what happened to an operation whose response was +// lost. A nil result means the mutation never committed, so it is safe to apply +// under the same operation id. This is the only correct answer to a timed-out +// COMMIT: retrying blindly could double-publish, and compensating with a DROP +// could delete a live catalog. +func (p *Publisher) ResolveOperation(ctx context.Context, operationID string) (*Result, error) { + if !operationIDPattern.MatchString(operationID) { + return nil, errors.New("catalog operation id is invalid") + } + ctx, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + var revision int64 + err := p.db.QueryRowContext(ctx, + `SELECT revision FROM trino_catalog_journal WHERE cell_id = $1 AND operation_id = $2`, + p.cellID, operationID).Scan(&revision) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("resolve catalog operation: %w", err) + } + return &Result{Revision: revision, Replayed: true}, nil +} + +// ResolveIntentSince reports whether a mutation with this exact intent +// committed AFTER the given revision. +// +// It exists because an operation id that carries the store's revision cannot be +// recomputed once the revision has moved: a caller that loses the COMMIT +// response and retries derives a DIFFERENT id, so resolving by id alone always +// misses the very case the journal is for. The intent - this catalog, this +// payload - plus "later than the revision I read before I tried" identifies the +// same commit without matching an older, identical publication of the same +// catalog. +func (p *Publisher) ResolveIntentSince(ctx context.Context, catalogName, payloadHash string, afterRevision int64) (*Result, error) { + if catalogName == "" || payloadHash == "" { + return nil, errors.New("resolving a catalog intent requires a catalog name and payload hash") + } + ctx, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + var revision int64 + err := p.db.QueryRowContext(ctx, + `SELECT revision FROM trino_catalog_journal + WHERE cell_id = $1 AND catalog_name = $2 AND payload_hash = $3 AND revision > $4 + ORDER BY revision DESC LIMIT 1`, + p.cellID, catalogName, payloadHash, afterRevision).Scan(&revision) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("resolve catalog intent: %w", err) + } + return &Result{Revision: revision, Replayed: true}, nil +} + +// checkFence implements root integration decision 2: the exact epoch AND the +// exact identity must match. A higher local epoch is not authority; it is a +// reason to call Takeover explicitly. +func (p *Publisher) checkFence(current State) error { + if current.WriterEpoch > p.epoch { + return fmt.Errorf("%w: recorded epoch %d is newer than %d", ErrFenced, current.WriterEpoch, p.epoch) + } + if current.WriterEpoch != p.epoch || current.Identity != p.identity { + return fmt.Errorf("%w: recorded writer is %q at epoch %d, this writer is %q at epoch %d", + ErrNotWriter, current.Identity, current.WriterEpoch, p.identity, p.epoch) + } + return nil +} + +// lockWriterState seeds and then locks the cell's writer row. The lock is what +// serializes every publisher of the cell, so it is taken before anything is +// read or decided. +func (p *Publisher) lockWriterState(ctx context.Context, tx *sql.Tx) (State, error) { + // Seed with the REAL row count, never a literal zero. + if _, err := tx.ExecContext(ctx, + `INSERT INTO trino_catalog_writer_state (cell_id, revision, writer_epoch, writer_identity, catalog_count) + SELECT $1::varchar, 0, 0, '', count(*) FROM trino_catalogs WHERE cell_id = $1 + ON CONFLICT (cell_id) DO NOTHING`, p.cellID); err != nil { + return State{}, fmt.Errorf("seed catalog writer state: %w", err) + } + var state State + if err := tx.QueryRowContext(ctx, + `SELECT revision, writer_epoch, writer_identity, catalog_count + FROM trino_catalog_writer_state WHERE cell_id = $1 FOR UPDATE`, p.cellID). + Scan(&state.Revision, &state.WriterEpoch, &state.Identity, &state.CatalogCount); err != nil { + return State{}, fmt.Errorf("lock catalog writer state: %w", err) + } + return state, nil +} + +type journalRecord struct { + revision int64 + payloadHash string +} + +func (p *Publisher) journalEntry(ctx context.Context, tx *sql.Tx, operationID string) (journalRecord, bool, error) { + var record journalRecord + err := tx.QueryRowContext(ctx, + `SELECT revision, payload_hash FROM trino_catalog_journal WHERE cell_id = $1 AND operation_id = $2`, + p.cellID, operationID).Scan(&record.revision, &record.payloadHash) + if errors.Is(err, sql.ErrNoRows) { + return journalRecord{}, false, nil + } + if err != nil { + return journalRecord{}, false, fmt.Errorf("read catalog journal: %w", err) + } + return record, true, nil +} + +// applyDefinition writes the definition change and returns the catalog version +// recorded in the journal (null for a removal). +func (p *Publisher) applyDefinition(ctx context.Context, tx *sql.Tx, mutation Mutation) (any, error) { + if mutation.Operation == OperationRemove { + if _, err := tx.ExecContext(ctx, + `DELETE FROM trino_catalogs WHERE cell_id = $1 AND catalog_name = $2`, + p.cellID, mutation.CatalogName); err != nil { + return nil, fmt.Errorf("remove catalog: %w", err) + } + return nil, nil + } + // Properties are stored verbatim, secret references included: Trino resolves + // ${ENV:...} per node, and resolving here would put a credential in the row. + properties, err := json.Marshal(mutation.Properties) + if err != nil { + return nil, fmt.Errorf("encode catalog properties: %w", err) + } + version := mutation.CatalogVersion() + if _, err := tx.ExecContext(ctx, + `INSERT INTO trino_catalogs (cell_id, catalog_name, connector_name, catalog_version, properties, updated_at) + VALUES ($1,$2,$3,$4,$5, now()) + ON CONFLICT (cell_id, catalog_name) DO UPDATE SET + connector_name = excluded.connector_name, + catalog_version = excluded.catalog_version, + properties = excluded.properties, + updated_at = now()`, + p.cellID, mutation.CatalogName, mutation.ConnectorName, version, string(properties)); err != nil { + return nil, fmt.Errorf("publish catalog: %w", err) + } + return version, nil +} + +// countCatalogs recomputes the row count inside the mutation transaction. An +// optimistic increment would drift from reality the moment anything else +// touched the table — during the migration bridge, exactly what happens. +func (p *Publisher) countCatalogs(ctx context.Context, tx *sql.Tx) (int, error) { + var count int + if err := tx.QueryRowContext(ctx, + `SELECT count(*) FROM trino_catalogs WHERE cell_id = $1`, p.cellID).Scan(&count); err != nil { + return 0, fmt.Errorf("count catalogs: %w", err) + } + return count, nil +} + +func (p *Publisher) inTransaction(ctx context.Context, fn func(*sql.Tx) error) error { + ctx, cancel := context.WithTimeout(ctx, p.timeout) + defer cancel() + tx, err := p.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return fmt.Errorf("begin catalog transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + // Bound the lock wait and each statement separately from the overall + // budget, so a wedged publisher cannot hold the cell's writer row. + if _, err := tx.ExecContext(ctx, `SET LOCAL lock_timeout = '`+lockTimeout+`'`); err != nil { + return fmt.Errorf("set lock timeout: %w", err) + } + if _, err := tx.ExecContext(ctx, `SET LOCAL statement_timeout = '`+statementTimeout+`'`); err != nil { + return fmt.Errorf("set statement timeout: %w", err) + } + if err := fn(tx); err != nil { + return err + } + if err := tx.Commit(); err != nil { + // The outcome is now UNKNOWN, not failed. The caller resolves it with + // ResolveOperation against the same operation id. + return fmt.Errorf("commit catalog transaction: %w", err) + } + committed = true + return nil +} + +func (m Mutation) validate() error { + if !operationIDPattern.MatchString(m.OperationID) || len(m.OperationID) > maxOperationIDLength { + return errors.New("catalog mutation requires a valid operation id") + } + if m.Operation != OperationAddOrReplace && m.Operation != OperationRemove { + return fmt.Errorf("unsupported catalog operation %q", m.Operation) + } + if !catalogNamePattern.MatchString(m.CatalogName) || len(m.CatalogName) > maxCatalogNameLength { + return errors.New("catalog mutation requires a valid catalog name") + } + if m.Operation == OperationRemove { + return nil + } + // Only validated connector names and properties are published; the store is + // read by every coordinator of the cell. + if m.ConnectorName == "" || len(m.ConnectorName) > maxConnectorLength || strings.ContainsAny(m.ConnectorName, " \t\n\r'\"") { + return errors.New("catalog mutation requires a valid connector name") + } + if len(m.Properties) > maxProperties { + return errors.New("catalog mutation declares too many properties") + } + for key, value := range m.Properties { + if key == "" || len(key) > 255 || strings.ContainsAny(key, " \t\n\r") { + return fmt.Errorf("catalog property %q has an invalid name", key) + } + if len(value) > maxPropertyValueLength || strings.ContainsAny(value, "\n\r") { + return fmt.Errorf("catalog property %q has an invalid value", key) + } + } + return nil +} + +// CatalogVersion is the content hash the coordinators compute for themselves. +// It has to match byte for byte or every unchanged catalog looks new. +func (m Mutation) CatalogVersion() string { + if m.Operation == OperationRemove { + return "" + } + return trinopool.CatalogVersion(m.CatalogName, m.ConnectorName, m.Properties) +} + +// PayloadHash identifies the INTENT of a mutation, so a retry of the same +// intent is recognizable as a replay and a different intent under the same +// operation id is recognizable as a conflict. The encoding is length-prefixed +// so no concatenation of fields can collide with another. +func (m Mutation) PayloadHash() string { + digest := sha256.New() + write := func(value string) { + length := make([]byte, 4) + binary.BigEndian.PutUint32(length, uint32(len(value))) + _, _ = digest.Write(length) + _, _ = digest.Write([]byte(value)) + } + write(m.Operation) + write(m.CatalogName) + write(m.ConnectorName) + keys := make([]string, 0, len(m.Properties)) + for key := range m.Properties { + keys = append(keys, key) + } + sort.Strings(keys) + length := make([]byte, 4) + binary.BigEndian.PutUint32(length, uint32(len(keys))) + _, _ = digest.Write(length) + for _, key := range keys { + write(key) + write(m.Properties[key]) + } + return hex.EncodeToString(digest.Sum(nil)) +} diff --git a/controlplane/trinogateway/backend.go b/controlplane/trinogateway/backend.go new file mode 100644 index 000000000..e53e10c8a --- /dev/null +++ b/controlplane/trinogateway/backend.go @@ -0,0 +1,87 @@ +package trinogateway + +import ( + "context" + "errors" + "fmt" + "net/http" +) + +// Legacy backend registration, which pooled registration depends on. +// +// PoolLifecycleService.registerMember looks the backend up by name and takes +// the member's endpoint from THAT record rather than from the request body. +// So a pooled instance needs a Gateway backend registration to exist before it +// can be registered as a member. This is a real prerequisite, not an internal +// detail, and it is implemented here rather than left as a hidden assumption +// that would surface as POOL_NOT_FOUND on the first spawn. +// +// The registration is created with active=false and is NEVER activated through +// the legacy route: `/gateway/backend/activate/{name}` would make the backend +// eligible for routing immediately, bypassing the whole certified-admission +// path. Eligibility comes only from the pooled protocol's admit call. + +// Backend is the legacy ProxyBackendConfiguration record. +type Backend struct { + Name string `json:"name"` + ProxyTo string `json:"proxyTo"` + ExternalURL string `json:"externalUrl,omitempty"` + RoutingGroup string `json:"routingGroup"` + // Active is the legacy routing switch. For a pooled member it stays false: + // the pooled protocol decides eligibility, and a backend flipped active by + // this path would serve tenant traffic with no certificate at all. + Active bool `json:"active"` +} + +// ListBackends reads every registered backend. +func (c *Client) ListBackends(ctx context.Context) ([]Backend, error) { + var backends []Backend + if err := c.do(ctx, http.MethodGet, "/gateway/backend/all", nil, &backends); err != nil { + return nil, err + } + return backends, nil +} + +// EnsureInactiveBackend creates the backend registration a pooled member needs, +// or verifies that an existing one matches. It refuses to touch a registration +// that is currently active or points somewhere else: that would either +// hijack another cluster's record or silently re-point live routing. +func (c *Client) EnsureInactiveBackend(ctx context.Context, backend Backend) error { + if backend.Name == "" || backend.ProxyTo == "" || backend.RoutingGroup == "" { + return errors.New("gateway backend registration requires a name, endpoint and routing group") + } + if backend.Active { + return errors.New("a pooled backend registration must be created inactive") + } + + existing, err := c.ListBackends(ctx) + if err != nil { + return err + } + for _, candidate := range existing { + if candidate.Name != backend.Name { + continue + } + // Already registered. Anything other than an exact, inactive match is + // a conflict the operator has to resolve, not something to overwrite. + if candidate.Active { + return fmt.Errorf("%w: backend %s is already active", ErrIdentityConflict, backend.Name) + } + if candidate.ProxyTo != backend.ProxyTo || candidate.RoutingGroup != backend.RoutingGroup { + return fmt.Errorf("%w: backend %s is registered with a different endpoint or routing group", + ErrIdentityConflict, backend.Name) + } + return nil + } + return c.do(ctx, http.MethodPost, "/gateway/backend/modify/add", backend, nil) +} + +// DeleteBackend removes a backend registration. It is called only after the +// Gateway has irreversibly retired the member that used it, so no routing +// decision can still reference it. +func (c *Client) DeleteBackend(ctx context.Context, name string) error { + if name == "" { + return errors.New("deleting a gateway backend requires a name") + } + return c.doRaw(ctx, http.MethodPost, "/gateway/backend/modify/delete", []byte(name), "text/plain", nil) +} diff --git a/controlplane/trinogateway/backend_test.go b/controlplane/trinogateway/backend_test.go new file mode 100644 index 000000000..862e68d81 --- /dev/null +++ b/controlplane/trinogateway/backend_test.go @@ -0,0 +1,89 @@ +package trinogateway + +import ( + "context" + "errors" + "net/http" + "testing" +) + +func poolBackend() Backend { + return Backend{ + Name: "pool-001-i-0007", ProxyTo: "https://i-0007.pool.invalid:8443", + RoutingGroup: "pool-001", Active: false, + } +} + +// Pooled registration reads the endpoint from the Gateway's own backend +// record, so the record has to exist first. Without this the first spawn fails +// with POOL_NOT_FOUND and the prerequisite stays invisible. +func TestEnsureInactiveBackendCreatesTheRegistration(t *testing.T) { + client, captured := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/gateway/backend/all" { + writeJSON(t, w, http.StatusOK, []Backend{}) + return + } + writeJSON(t, w, http.StatusOK, map[string]any{}) + }) + if err := client.EnsureInactiveBackend(context.Background(), poolBackend()); err != nil { + t.Fatalf("ensure backend: %v", err) + } + created := (*captured)[1] + if created.method != http.MethodPost || created.path != "/gateway/backend/modify/add" { + t.Fatalf("request = %s %s", created.method, created.path) + } + // Creating it active would make it eligible for tenant routing immediately, + // with no certificate and no admission - exactly what the pooled protocol + // exists to prevent. + if created.body["active"] != false { + t.Fatalf("backend was registered with active=%v", created.body["active"]) + } + for _, field := range []string{"name", "proxyTo", "routingGroup"} { + if _, present := created.body[field]; !present { + t.Errorf("registration is missing %q", field) + } + } +} + +func TestEnsureInactiveBackendIsIdempotent(t *testing.T) { + client, captured := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, []Backend{poolBackend()}) + }) + if err := client.EnsureInactiveBackend(context.Background(), poolBackend()); err != nil { + t.Fatalf("ensure backend: %v", err) + } + if len(*captured) != 1 { + t.Fatalf("an existing registration triggered %d requests", len(*captured)) + } +} + +// Adopting somebody else's registration would re-point live routing or hijack +// another cluster's record. +func TestEnsureInactiveBackendRefusesAConflictingRegistration(t *testing.T) { + cases := map[string]Backend{ + "already active": {Name: "pool-001-i-0007", ProxyTo: "https://i-0007.pool.invalid:8443", RoutingGroup: "pool-001", Active: true}, + "other endpoint": {Name: "pool-001-i-0007", ProxyTo: "https://somebody-else.invalid:8443", RoutingGroup: "pool-001"}, + "other group": {Name: "pool-001-i-0007", ProxyTo: "https://i-0007.pool.invalid:8443", RoutingGroup: "adhoc"}, + } + for name, existing := range cases { + t.Run(name, func(t *testing.T) { + client, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, []Backend{existing}) + }) + if err := client.EnsureInactiveBackend(context.Background(), poolBackend()); !errors.Is(err, ErrIdentityConflict) { + t.Fatalf("error = %v, want ErrIdentityConflict", err) + } + }) + } +} + +func TestEnsureInactiveBackendRefusesToRegisterAnActiveBackend(t *testing.T) { + client, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, []Backend{}) + }) + active := poolBackend() + active.Active = true + if err := client.EnsureInactiveBackend(context.Background(), active); err == nil { + t.Fatal("an active pooled registration was accepted") + } +} diff --git a/controlplane/trinogateway/client.go b/controlplane/trinogateway/client.go new file mode 100644 index 000000000..0ecf8a725 --- /dev/null +++ b/controlplane/trinogateway/client.go @@ -0,0 +1,346 @@ +package trinogateway + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + "unicode" +) + +const ( + basePath = "/gateway/v1/pools" + + // One admin request budget, per the agreed retry policy. Operation progress + // deadlines are the caller's and are deliberately much longer: a slow drain + // is not a slow request. + defaultRequestTimeout = 10 * time.Second + maxResponseBytes = 1 << 20 + minAdminTokenLength = 32 +) + +// Config configures the client. The credential is the Gateway's EXISTING admin +// token; this protocol introduces no new secret and no client certificate. +type Config struct { + BaseURL string + AdminToken string + TLSServerName string + // AllowPlaintext permits an http:// origin. Only tests set it: the token + // would otherwise cross the network in the clear. + AllowPlaintext bool + RequestTimeout time.Duration + HTTPClient *http.Client +} + +// Client speaks the pooled member lifecycle protocol. +type Client struct { + baseURL string + token string + http *http.Client + timeout time.Duration +} + +// NewClient validates the origin and credential up front, so a misconfigured +// deployment fails at startup rather than at the first drain. +func NewClient(config Config) (*Client, error) { + origin, err := url.Parse(strings.TrimSpace(config.BaseURL)) + if err != nil || origin.Hostname() == "" || origin.User != nil || + origin.RawQuery != "" || origin.Fragment != "" || + (origin.Path != "" && origin.Path != "/") { + return nil, errors.New("gateway client requires a credential-free origin without a path") + } + plaintextAllowed := config.AllowPlaintext && origin.Scheme == "http" + if origin.Scheme != "https" && !plaintextAllowed { + return nil, errors.New("gateway client requires an HTTPS origin") + } + token := strings.TrimSpace(config.AdminToken) + if len(token) < minAdminTokenLength || strings.IndexFunc(token, unicode.IsControl) != -1 { + return nil, errors.New("gateway client requires the existing admin token") + } + + client := config.HTTPClient + if client == nil { + transport := http.DefaultTransport.(*http.Transport).Clone() + // A registry-owned internal endpoint: no proxy, pinned server name. + transport.Proxy = nil + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, ServerName: config.TLSServerName} + transport.MaxIdleConnsPerHost = 4 + transport.ResponseHeaderTimeout = 5 * time.Second + client = &http.Client{ + Transport: transport, + // Redirects are refused: following one would replay an admin + // mutation against an origin nobody validated. + CheckRedirect: func(*http.Request, []*http.Request) error { + return errors.New("gateway client does not follow redirects") + }, + } + } + timeout := config.RequestTimeout + if timeout <= 0 { + timeout = defaultRequestTimeout + } + return &Client{ + baseURL: strings.TrimSuffix(origin.String(), "/"), + token: token, + http: client, + timeout: timeout, + }, nil +} + +// GetPool reads the pool's authoritative counts and generations. +func (c *Client) GetPool(ctx context.Context, poolID string) (PoolState, error) { + var pool PoolState + err := c.do(ctx, http.MethodGet, c.poolPath(poolID), nil, &pool) + return pool, err +} + +// ConfigurePool applies the desired pool specification. +func (c *Client) ConfigurePool(ctx context.Context, poolID string, request ConfigurePoolRequest) (PoolState, error) { + var pool PoolState + err := c.do(ctx, http.MethodPut, c.poolPath(poolID), request, &pool) + return pool, err +} + +// ListMembers reads the authoritative member list. The Gateway returns a bare +// JSON array, not an envelope. +func (c *Client) ListMembers(ctx context.Context, poolID string) ([]Member, error) { + var members []Member + if err := c.do(ctx, http.MethodGet, c.poolPath(poolID)+"/members", nil, &members); err != nil { + return nil, err + } + return members, nil +} + +// GetMember reads one member back. This is the read-back path after a lost +// response on a member mutation. +func (c *Client) GetMember(ctx context.Context, poolID, instanceID string) (Member, error) { + var member Member + err := c.do(ctx, http.MethodGet, c.memberPath(poolID, instanceID), nil, &member) + return member, err +} + +// GetObligations reads what still pins a member. Drain completion is decided +// here and nowhere else. +func (c *Client) GetObligations(ctx context.Context, poolID, instanceID string) (Obligations, error) { + var obligations Obligations + err := c.do(ctx, http.MethodGet, c.memberPath(poolID, instanceID)+"/obligations", nil, &obligations) + return obligations, err +} + +// RegisterMember creates a PREPARING member against an EXISTING Gateway backend +// registration. It is not eligible for tenant work. +func (c *Client) RegisterMember(ctx context.Context, poolID string, request RegisterMemberRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.poolPath(poolID)+"/members", request) +} + +// AdmitMember is the single certified-activation call: it carries the +// duckgres-performed validation receipt and the generation CAS. The Gateway +// enforces certificate freshness, the budgets and any open publication barrier, +// and independently verifies the live process identity. +func (c *Client) AdmitMember(ctx context.Context, poolID, instanceID string, request AdmitMemberRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.memberPath(poolID, instanceID)+"/admit", request) +} + +// DrainMember starts a planned drain. The Gateway refuses it when the serving +// floor would break; that refusal is authoritative and is never overridden. +func (c *Client) DrainMember(ctx context.Context, poolID, instanceID string, request MemberStepRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.memberPath(poolID, instanceID)+"/drain", request) +} + +// SealMember moves a drained member to SEALED. The Gateway checks that no +// obligations remain; duckgres never decides drain completion from a timer. +func (c *Client) SealMember(ctx context.Context, poolID, instanceID string, request MemberStepRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.memberPath(poolID, instanceID)+"/seal", request) +} + +// SuspectMember excludes a member from new admissions. It authorizes nothing +// destructive. +func (c *Client) SuspectMember(ctx context.Context, poolID, instanceID string, request SuspectMemberRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.memberPath(poolID, instanceID)+"/suspect", request) +} + +// LostMember records that a member's process terminated, with evidence. +func (c *Client) LostMember(ctx context.Context, poolID, instanceID string, request LostMemberRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.memberPath(poolID, instanceID)+"/lost", request) +} + +// RetireMember claims retirement for the exact incarnation. This is the +// irreversible step, and its receipt is what authorizes deleting Kubernetes +// objects. Nothing else does. +func (c *Client) RetireMember(ctx context.Context, poolID, instanceID string, request MemberStepRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.memberPath(poolID, instanceID)+"/retire", request) +} + +// MemberRetired reports that the resources are gone. The Gateway records the +// assertion; it never infers deletion for itself. +func (c *Client) MemberRetired(ctx context.Context, poolID, instanceID string, request MemberStepRequest) (Member, error) { + return c.member(ctx, http.MethodPost, c.memberPath(poolID, instanceID)+"/retired", request) +} + +// GetFailureReceipt reads a failed member's preserved obligations. +func (c *Client) GetFailureReceipt(ctx context.Context, poolID, instanceID string) (FailureReceipt, error) { + var receipt FailureReceipt + err := c.do(ctx, http.MethodGet, c.memberPath(poolID, instanceID)+"/failure-receipt", nil, &receipt) + return receipt, err +} + +// GetOperation reads recorded step outcomes. This is the ONLY correct response +// to a lost reply: the operator resolves what actually happened under the same +// operation id instead of minting a new one. +func (c *Client) GetOperation(ctx context.Context, poolID, operationID string) (OperationHistory, error) { + var history OperationHistory + err := c.do(ctx, http.MethodGet, c.poolPath(poolID)+"/operations/"+url.PathEscape(operationID), nil, &history) + return history, err +} + +// OpenPublication opens the tenant publication barrier. +func (c *Client) OpenPublication(ctx context.Context, poolID string, request OpenPublicationRequest) (Publication, error) { + var publication Publication + err := c.do(ctx, http.MethodPost, c.poolPath(poolID)+"/publications", request, &publication) + return publication, err +} + +// RecordPublicationReceipt records one member's applied revision. +func (c *Client) RecordPublicationReceipt(ctx context.Context, poolID, publicationID string, request PublicationReceiptRequest) (Publication, error) { + var publication Publication + err := c.do(ctx, http.MethodPost, c.publicationPath(poolID, publicationID)+"/receipts", request, &publication) + return publication, err +} + +// CommitPublication opens the tenant admission gate. After it commits the +// Gateway's receipt is authoritative even if duckgres has not checkpointed yet; +// recovery completes the checkpoint and never retracts an opened gate. +func (c *Client) CommitPublication(ctx context.Context, poolID, publicationID string, request CommitPublicationRequest) (Publication, error) { + var publication Publication + err := c.do(ctx, http.MethodPost, c.publicationPath(poolID, publicationID)+"/commit", request, &publication) + return publication, err +} + +// AbandonPublication gives up a barrier that never committed. +func (c *Client) AbandonPublication(ctx context.Context, poolID, publicationID string, step Step) (Publication, error) { + var publication Publication + err := c.do(ctx, http.MethodPost, c.publicationPath(poolID, publicationID)+"/abandon", step, &publication) + return publication, err +} + +// GetPublication reads a barrier back, which is how a lost commit response is +// resolved. +func (c *Client) GetPublication(ctx context.Context, poolID, publicationID string) (Publication, error) { + var publication Publication + err := c.do(ctx, http.MethodGet, c.publicationPath(poolID, publicationID), nil, &publication) + return publication, err +} + +// GetTenant reads a tenant's admission gate. +func (c *Client) GetTenant(ctx context.Context, poolID, tenant string) (TenantAdmission, error) { + var admission TenantAdmission + err := c.do(ctx, http.MethodGet, c.tenantPath(poolID, tenant), nil, &admission) + return admission, err +} + +// PublishTenantPrincipals publishes the authoritative principal to tenant +// binding the admission restriction keys on. +// +// The route is PUT, matching PoolResource: the call replaces the tenant's whole +// principal set rather than appending to it, and a repeat of the same set is the +// same state. Sending it as a POST reached no route at all. +func (c *Client) PublishTenantPrincipals(ctx context.Context, poolID, tenant string, request PublishPrincipalsRequest) (TenantAdmission, error) { + var admission TenantAdmission + err := c.do(ctx, http.MethodPut, c.tenantPath(poolID, tenant)+"/principals", request, &admission) + return admission, err +} + +// RevokeTenant closes a tenant's admission gate. +func (c *Client) RevokeTenant(ctx context.Context, poolID, tenant string, request RevokeTenantRequest) (TenantAdmission, error) { + var admission TenantAdmission + err := c.do(ctx, http.MethodDelete, c.tenantPath(poolID, tenant), request, &admission) + return admission, err +} + +func (c *Client) member(ctx context.Context, method, path string, body any) (Member, error) { + var member Member + err := c.do(ctx, method, path, body, &member) + return member, err +} + +func (c *Client) poolPath(poolID string) string { + return basePath + "/" + url.PathEscape(poolID) +} + +func (c *Client) memberPath(poolID, instanceID string) string { + return c.poolPath(poolID) + "/members/" + url.PathEscape(instanceID) +} + +func (c *Client) publicationPath(poolID, publicationID string) string { + return c.poolPath(poolID) + "/publications/" + url.PathEscape(publicationID) +} + +func (c *Client) tenantPath(poolID, tenant string) string { + return c.poolPath(poolID) + "/tenants/" + url.PathEscape(tenant) +} + +func (c *Client) do(ctx context.Context, method, path string, body, target any) error { + var encoded []byte + if body != nil { + var err error + encoded, err = json.Marshal(body) + if err != nil { + return fmt.Errorf("encode gateway request: %w", err) + } + } + return c.doRaw(ctx, method, path, encoded, "application/json", target) +} + +// doRaw sends an already-encoded body. The legacy backend-delete endpoint takes +// a bare string rather than JSON, which is why the encoding is a parameter. +func (c *Client) doRaw(ctx context.Context, method, path string, body []byte, contentType string, target any) error { + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + var payload io.Reader + if body != nil { + payload = bytes.NewReader(body) + } + request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, payload) + if err != nil { + return fmt.Errorf("build gateway request: %w", err) + } + // The Gateway accepts either form of the existing admin credential; sending + // both keeps the client working across its own auth refactors. + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("X-Gateway-Transaction-Admin-Token", c.token) + request.Header.Set("Accept", "application/json") + if body != nil { + request.Header.Set("Content-Type", contentType) + } + + response, err := c.http.Do(request) + if err != nil { + // No verdict: the mutation may or may not have been applied. The caller + // must resolve this through GetOperation rather than retrying blind. + return fmt.Errorf("gateway request failed: %w", err) + } + defer func() { _ = response.Body.Close() }() + + raw, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1)) + if err != nil || len(raw) > maxResponseBytes { + return errors.New("gateway response exceeds the size limit") + } + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusCreated { + return newGatewayError(response.StatusCode, response.Header.Get("X-Trino-Gateway-Error"), strings.TrimSpace(string(raw))) + } + if target == nil { + return nil + } + if err := json.Unmarshal(raw, target); err != nil { + return fmt.Errorf("decode gateway response: %w", err) + } + return nil +} diff --git a/controlplane/trinogateway/client_test.go b/controlplane/trinogateway/client_test.go new file mode 100644 index 000000000..a617db912 --- /dev/null +++ b/controlplane/trinogateway/client_test.go @@ -0,0 +1,296 @@ +package trinogateway + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +type capturedRequest struct { + method string + path string + body map[string]any + header http.Header +} + +func newTestClient(t *testing.T, handler http.HandlerFunc) (*Client, *[]capturedRequest) { + t.Helper() + var captured []capturedRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + record := capturedRequest{method: r.Method, path: r.URL.Path, header: r.Header.Clone()} + if payload, err := io.ReadAll(r.Body); err == nil && len(payload) > 0 { + _ = json.Unmarshal(payload, &record.body) + } + captured = append(captured, record) + handler(w, r) + })) + t.Cleanup(server.Close) + + client, err := NewClient(Config{BaseURL: server.URL, AdminToken: "0123456789abcdef0123456789abcdef", AllowPlaintext: true}) + if err != nil { + t.Fatalf("new client: %v", err) + } + return client, &captured +} + +func writeJSON(t *testing.T, w http.ResponseWriter, status int, value any) { + t.Helper() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(value); err != nil { + t.Fatalf("encode response: %v", err) + } +} + +func gatewayError(w http.ResponseWriter, status int, code string) { + w.Header().Set("X-Trino-Gateway-Error", code) + w.WriteHeader(status) + _, _ = w.Write([]byte(code)) +} + +// memberResponse is the real Java-serialized member fixture, adjusted for the +// phase and generation a given test needs. Building it from the fixture keeps +// the handler's responses in the shape the Gateway actually emits. +func memberResponse(t *testing.T, phase string, generation int64) map[string]any { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "member.json")) + if err != nil { + t.Fatalf("read member fixture: %v", err) + } + var response map[string]any + if err := json.Unmarshal(data, &response); err != nil { + t.Fatalf("decode member fixture: %v", err) + } + response["phase"], response["generation"] = phase, generation + return response +} + +// The Gateway's existing admin credential is reused; this adds no new secret. +func TestClientSendsTheExistingAdminCredential(t *testing.T) { + client, captured := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, memberResponse(t, "PREPARING", 1)) + }) + if _, err := client.RegisterMember(context.Background(), "pool-1", RegisterMemberRequest{ + Step: Step{OperationID: "op-1", StepID: "register", ControllerEpoch: 7}, + InstanceID: "i-1", BackendName: "pool-1-i-1", URL: "https://i-1.invalid:8443", + PodUID: "pod-uid", BootID: "boot-id", ConfigRevision: "r-42", + }); err != nil { + t.Fatalf("register: %v", err) + } + request := (*captured)[0] + if request.method != http.MethodPost || request.path != "/gateway/v1/pools/pool-1/members" { + t.Fatalf("request = %s %s", request.method, request.path) + } + if request.header.Get("Authorization") != "Bearer 0123456789abcdef0123456789abcdef" { + t.Fatalf("missing bearer credential: %q", request.header.Get("Authorization")) + } + if request.header.Get("X-Gateway-Transaction-Admin-Token") != "0123456789abcdef0123456789abcdef" { + t.Fatal("missing admin token header") + } +} + +// Field spellings are the cross-repo contract; a typo here fails closed at +// runtime and would only show up in a live deployment. +func TestRegisterMemberBodyMatchesTheContract(t *testing.T) { + client, captured := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, memberResponse(t, "PREPARING", 1)) + }) + if _, err := client.RegisterMember(context.Background(), "pool-1", RegisterMemberRequest{ + Step: Step{OperationID: "op-1", StepID: "register", ControllerEpoch: 7}, + InstanceID: "i-1", BackendName: "pool-1-i-1", URL: "https://i-1.invalid:8443", + PodUID: "pod-uid", BootID: "boot-id", ConfigRevision: "r-42", + }); err != nil { + t.Fatalf("register: %v", err) + } + body := (*captured)[0].body + // Exactly the fields PoolLifecycleService.registerMember reads. + for _, field := range []string{"operationId", "stepId", "controllerEpoch", "instanceId", "backendName", "url", "podUid", "bootId", "configRevision"} { + if _, present := body[field]; !present { + t.Errorf("request body is missing %q", field) + } + } + // The Gateway computes the guard's payload hash from the canonicalized body + // itself. Sending one would change that body and therefore the hash, making + // an identical replay look like a changed intent. + if _, present := body["payloadHash"]; present { + t.Error("the client sent a payloadHash the Gateway computes itself") + } + // The endpoint comes from the Gateway's own backend registration; there is + // no externalUrl input. + if _, present := body["externalUrl"]; present { + t.Error("the client sent externalUrl, which the Gateway does not read") + } +} + +func TestAdmitIsASingleCallCarryingTheReceipt(t *testing.T) { + client, captured := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, memberResponse(t, "ACTIVE", 4)) + }) + member, err := client.AdmitMember(context.Background(), "pool-1", "i-1", admitRequest()) + if err != nil { + t.Fatalf("admit: %v", err) + } + if member.Phase != "ACTIVE" || member.Generation != 4 { + t.Fatalf("member = %+v", member) + } + request := (*captured)[0] + // Admission is ONE call with a nested receipt. There is no /certificate and + // no /activate route; posting to either would 404 in production while every + // mock-based test kept passing. + if request.path != "/gateway/v1/pools/pool-1/members/i-1/admit" { + t.Fatalf("path = %s", request.path) + } + if request.body["expectedGeneration"] != float64(3) { + t.Fatalf("expectedGeneration = %v", request.body["expectedGeneration"]) + } + receipt, ok := request.body["receipt"].(map[string]any) + if !ok { + t.Fatalf("receipt is not a nested object: %v", request.body["receipt"]) + } + // Every one of these is read with a required-text accessor on the Java + // side: an empty value is a 400, not a default. + for _, field := range []string{"certificateHash", "configRevision", "authRevision", "podUid", "bootId", "nodeId", "coordinatorId", "readyWorkers", "checks"} { + if _, present := receipt[field]; !present { + t.Errorf("receipt is missing %q", field) + } + } +} + +func admitRequest() AdmitMemberRequest { + return AdmitMemberRequest{ + Step: Step{OperationID: "op-1", StepID: "admit", ControllerEpoch: 7}, + ExpectedGeneration: 3, + Receipt: ValidationReceipt{ + CertificateHash: "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff", + ConfigRevision: "r-42", AuthRevision: "auth-9", + PodUID: "pod-uid", BootID: "boot-id", NodeID: "node-1", CoordinatorID: "abcde", + ReadyWorkers: 4, + Checks: []string{CheckImage, CheckWorkers, CheckCatalogRevision, CheckAuthRevision, CheckOperationalConnection}, + }, + } +} + +// Terminal conflicts are surfaced to the operator, never hot-retried: retrying +// a stale epoch or a changed intent cannot succeed and hides a real fault. +func TestGatewayConflictsAreTypedAndTerminal(t *testing.T) { + cases := map[string]error{ + "POOL_STALE_EPOCH": ErrStaleEpoch, + "POOL_INTENT_CHANGED": ErrIntentChanged, + "POOL_STALE_GENERATION": ErrStaleGeneration, + "POOL_PHASE": ErrPhase, + "POOL_IRREVERSIBLE": ErrIrreversible, + "POOL_SERVING_FLOOR": ErrServingFloor, + "POOL_SURGE_BUDGET": ErrSurgeBudget, + "POOL_NOT_CERTIFIED": ErrNotCertified, + "POOL_PUBLICATION_BARRIER": ErrPublicationBarrier, + "POOL_MEMBERSHIP_CHANGED": ErrMembershipChanged, + "POOL_RECEIPTS_INCOMPLETE": ErrReceiptsIncomplete, + "POOL_EVIDENCE_REQUIRED": ErrEvidenceRequired, + "POOL_DISABLED": ErrPoolDisabled, + "POOL_VALIDATION": ErrValidation, + "POOL_NOT_FOUND": ErrNotFound, + "POOL_IDENTITY_CONFLICT": ErrIdentityConflict, + "POOL_APIMODE": ErrAPIMode, + "POOL_NOT_DRAINED": ErrNotDrained, + "POOL_REPAIR_BUDGET": ErrRepairBudget, + } + for code, expected := range cases { + t.Run(code, func(t *testing.T) { + status := http.StatusConflict + switch code { + case "POOL_DISABLED", "POOL_NOT_FOUND": + status = http.StatusNotFound + case "POOL_VALIDATION": + status = http.StatusBadRequest + } + client, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gatewayError(w, status, code) + }) + _, err := client.AdmitMember(context.Background(), "pool-1", "i-1", admitRequest()) + if !errors.Is(err, expected) { + t.Fatalf("error = %v, want %v", err, expected) + } + if Retryable(err) { + t.Fatalf("%s was classified as retryable", code) + } + }) + } +} + +// A 503 is a transient condition, not a verdict about the operation. +func TestUnavailableIsRetryable(t *testing.T) { + client, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + gatewayError(w, http.StatusServiceUnavailable, "ROUTING_STATE_UNAVAILABLE") + }) + _, err := client.GetPool(context.Background(), "pool-1") + if err == nil || !Retryable(err) { + t.Fatalf("error = %v, want a retryable failure", err) + } +} + +// A replayed step returns the RECORDED result. The operator must be able to +// tell that apart from a fresh mutation so it does not double-count effects. +func TestReplayedResponsesAreReported(t *testing.T) { + client, _ := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + response := memberResponse(t, "ACTIVE", 4) + response["replayed"] = true + writeJSON(t, w, http.StatusOK, response) + }) + member, err := client.AdmitMember(context.Background(), "pool-1", "i-1", admitRequest()) + if err != nil { + t.Fatalf("admit: %v", err) + } + if !member.Replayed { + t.Fatal("a replayed response was reported as a fresh mutation") + } +} + +// A lost response is resolved by reading the SAME operation back, never by +// minting a fresh operation id. +func TestOperationReadBackResolvesALostResponse(t *testing.T) { + client, captured := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { + writeJSON(t, w, http.StatusOK, map[string]any{ + "protocolVersion": 1, "operationId": "op-1", + "steps": []any{map[string]any{ + "stepId": "admit", "payloadHash": "abc", "controllerEpoch": 7, + "outcome": "OK", "recordedAt": "2026-09-18T00:00:00Z", + "result": map[string]any{"phase": "ACTIVE"}, + }}, + }) + }) + history, err := client.GetOperation(context.Background(), "pool-1", "op-1") + if err != nil { + t.Fatalf("get operation: %v", err) + } + if len(history.Steps) != 1 || history.Steps[0].StepID != "admit" || history.Steps[0].Outcome != "OK" { + t.Fatalf("history = %+v", history) + } + if (*captured)[0].path != "/gateway/v1/pools/pool-1/operations/op-1" { + t.Fatalf("path = %s", (*captured)[0].path) + } +} + +// A plaintext or credential-bearing origin must be refused: this client carries +// an admin token. +func TestClientRejectsUnsafeConfiguration(t *testing.T) { + cases := map[string]Config{ + "plaintext origin": {BaseURL: "http://gateway.invalid", AdminToken: "0123456789abcdef0123456789abcdef"}, + "credentials in origin": {BaseURL: "https://user:pass@gateway.invalid", AdminToken: "0123456789abcdef0123456789abcdef", AllowPlaintext: true}, + "short token": {BaseURL: "https://gateway.invalid", AdminToken: "too-short"}, + "missing token": {BaseURL: "https://gateway.invalid"}, + "path in origin": {BaseURL: "https://gateway.invalid/api", AdminToken: "0123456789abcdef0123456789abcdef"}, + } + for name, config := range cases { + t.Run(name, func(t *testing.T) { + if _, err := NewClient(config); err == nil { + t.Fatalf("accepted %s", name) + } + }) + } +} diff --git a/controlplane/trinogateway/errors.go b/controlplane/trinogateway/errors.go new file mode 100644 index 000000000..0260086b6 --- /dev/null +++ b/controlplane/trinogateway/errors.go @@ -0,0 +1,132 @@ +// Package trinogateway is the client for the Gateway's pooled member lifecycle +// protocol (v1). The Gateway owns member eligibility, admission, obligation +// accounting, the minimum-serving floor, the publication barrier and the +// irreversible retirement claim; duckgres owns Kubernetes effects and the +// durable operations that drive them. There is no distributed transaction +// between the two, so this client is built around one rule: a lost response is +// UNKNOWN, and unknown is resolved by reading the same operation back. +package trinogateway + +import ( + "errors" + "fmt" + "net/http" +) + +// Typed conflicts, mapped from the Gateway's X-Trino-Gateway-Error header. +// Each one is terminal for the step: retrying a stale epoch, a changed intent +// or an exhausted budget cannot succeed, and hot-retrying would bury a real +// fault under a retry loop. +var ( + ErrPoolDisabled = errors.New("gateway pool protocol is disabled") + ErrStaleEpoch = errors.New("gateway rejected a stale controller epoch") + ErrIntentChanged = errors.New("gateway recorded a different payload for this operation step") + ErrStaleGeneration = errors.New("gateway rejected a stale generation") + ErrPhase = errors.New("gateway refused the transition from the member's current phase") + ErrIrreversible = errors.New("gateway refused to resume or reuse a retiring identity") + ErrServingFloor = errors.New("gateway refused a drain that would breach the serving floor") + ErrSurgeBudget = errors.New("gateway refused an activation: surge or repair budget exhausted") + ErrNotCertified = errors.New("gateway refused an activation: missing, stale or mismatched certificate") + ErrPublicationBarrier = errors.New("gateway refused a join without a receipt at the open publication revision") + ErrMembershipChanged = errors.New("gateway membership generation moved during the publication") + ErrReceiptsIncomplete = errors.New("gateway refused a publication commit: a receipt is missing") + ErrEvidenceRequired = errors.New("gateway refused a loss claim without termination evidence") + ErrTenantNotAdmitted = errors.New("gateway tenant admission gate is not open") + ErrUnavailable = errors.New("gateway is unavailable") + + // The Gateway rejects a malformed request body with POOL_VALIDATION. For + // this client that is a bug in the caller, never something to retry. + ErrValidation = errors.New("gateway rejected the request body") + ErrNotFound = errors.New("gateway does not know this pool, member or receipt") + // ErrIdentityConflict covers a member registered against a backend that + // belongs to another routing group, or an endpoint that does not match the + // Gateway's own backend registration. + ErrIdentityConflict = errors.New("gateway refused a conflicting member identity") + ErrAPIMode = errors.New("gateway refused the call for this pool's api mode") + // ErrNotDrained is the Gateway refusing to seal a member that still has + // obligations. It is the authoritative answer to "is the drain finished". + ErrNotDrained = errors.New("gateway refused to seal a member that is not drained") + ErrRepairBudget = errors.New("gateway refused an activation: repair budget exhausted") + // ErrPrincipalConflict means a published principal already belongs to + // another tenant in this pool. That is an ambiguity the gate must never + // resolve by guessing, so the publication is refused whole. + ErrPrincipalConflict = errors.New("gateway refused a principal already bound to another tenant") +) + +// Error carries the Gateway's response code alongside the mapped sentinel. +type Error struct { + Code string + Status int + Body string + err error +} + +func (e *Error) Error() string { + if e.Code == "" { + return fmt.Sprintf("gateway returned HTTP %d", e.Status) + } + return fmt.Sprintf("gateway returned %s (HTTP %d)", e.Code, e.Status) +} + +func (e *Error) Unwrap() error { return e.err } + +// codeSentinels is the whole mapping. A code the Gateway adds later is +// deliberately NOT retryable: an unrecognized refusal is treated as terminal +// for the step, so an unknown rejection cannot become an infinite retry. +var codeSentinels = map[string]error{ + "POOL_DISABLED": ErrPoolDisabled, + "POOL_STALE_EPOCH": ErrStaleEpoch, + "POOL_INTENT_CHANGED": ErrIntentChanged, + "POOL_STALE_GENERATION": ErrStaleGeneration, + "POOL_PHASE": ErrPhase, + "POOL_IRREVERSIBLE": ErrIrreversible, + "POOL_SERVING_FLOOR": ErrServingFloor, + "POOL_SURGE_BUDGET": ErrSurgeBudget, + "POOL_NOT_CERTIFIED": ErrNotCertified, + "POOL_PUBLICATION_BARRIER": ErrPublicationBarrier, + "POOL_MEMBERSHIP_CHANGED": ErrMembershipChanged, + "POOL_RECEIPTS_INCOMPLETE": ErrReceiptsIncomplete, + "POOL_EVIDENCE_REQUIRED": ErrEvidenceRequired, + "POOL_VALIDATION": ErrValidation, + "POOL_NOT_FOUND": ErrNotFound, + "POOL_IDENTITY_CONFLICT": ErrIdentityConflict, + "POOL_APIMODE": ErrAPIMode, + "POOL_NOT_DRAINED": ErrNotDrained, + "POOL_REPAIR_BUDGET": ErrRepairBudget, + "POOL_PRINCIPAL_CONFLICT": ErrPrincipalConflict, + "TENANT_NOT_ADMITTED": ErrTenantNotAdmitted, + "ROUTING_STATE_UNAVAILABLE": ErrUnavailable, + "TENANT_IDENTITY_UNVERIFIED": ErrTenantNotAdmitted, +} + +func newGatewayError(status int, code, body string) error { + mapped := codeSentinels[code] + if mapped == nil { + switch { + case status == http.StatusServiceUnavailable, status == http.StatusTooManyRequests, status >= 500: + mapped = ErrUnavailable + default: + mapped = errors.New("gateway refused the request") + } + } + return &Error{Code: code, Status: status, Body: body, err: mapped} +} + +// Retryable reports whether an error is a transient condition rather than a +// verdict about the operation. Only availability failures qualify: every +// conflict above means the Gateway made a decision that a retry cannot change. +// +// Note what this does NOT say: a transport error has no verdict at all. The +// caller must resolve those through GetOperation before retrying, because the +// mutation may well have been applied. +func Retryable(err error) bool { + return errors.Is(err, ErrUnavailable) +} + +// IsNotFound reports the Gateway answering that it has no such pool, member or +// publication. It is a DECISION - the thing is absent - as opposed to an +// unanswered call, so a caller may act on it rather than resolving it by +// read-back. +func IsNotFound(err error) bool { + return errors.Is(err, ErrNotFound) +} diff --git a/controlplane/trinogateway/testdata/failure_receipt.json b/controlplane/trinogateway/testdata/failure_receipt.json new file mode 100644 index 000000000..8a114a7f1 --- /dev/null +++ b/controlplane/trinogateway/testdata/failure_receipt.json @@ -0,0 +1,14 @@ +{ + "protocolVersion" : 1, + "poolId" : "pool-001", + "instanceId" : "i-0007", + "incarnation" : "11111111-1111-4111-8111-111111111111", + "evidence" : "PROCESS_TERMINATED", + "detail" : { + "source" : "kubernetes-pod-absent" + }, + "outstandingAdmissions" : 0, + "outstandingTransactions" : 1, + "outstandingQueries" : 2, + "recordedAt" : "2026-09-18T00:00:00Z" +} diff --git a/controlplane/trinogateway/testdata/member.json b/controlplane/trinogateway/testdata/member.json new file mode 100644 index 000000000..72f1d0211 --- /dev/null +++ b/controlplane/trinogateway/testdata/member.json @@ -0,0 +1,30 @@ +{ + "protocolVersion" : 1, + "poolId" : "pool-001", + "instanceId" : "i-0007", + "incarnation" : "11111111-1111-4111-8111-111111111111", + "backendName" : "pool-001-i-0007", + "url" : "https://i-0007.pool.invalid:8443", + "externalUrl" : "https://i-0007.external.invalid:8443", + "phase" : "ACTIVE", + "generation" : 4, + "controllerEpoch" : 7, + "podUid" : "pod-uid-0007", + "bootId" : "boot-0007", + "nodeId" : "node-0007", + "coordinatorId" : "abcde", + "configRevision" : "r-42", + "certifiedRevision" : "r-42", + "authRevision" : "auth-9", + "repair" : false, + "repairFor" : null, + "retirementKind" : null, + "pendingRequests" : 0, + "openTransactions" : 0, + "activeQueries" : 0, + "readyToSeal" : false, + "drained" : false, + "eligible" : true, + "membershipGeneration" : 19, + "replayed" : false +} diff --git a/controlplane/trinogateway/testdata/members.json b/controlplane/trinogateway/testdata/members.json new file mode 100644 index 000000000..e9740b64b --- /dev/null +++ b/controlplane/trinogateway/testdata/members.json @@ -0,0 +1,30 @@ +[ { + "protocolVersion" : 1, + "poolId" : "pool-001", + "instanceId" : "i-0007", + "incarnation" : "11111111-1111-4111-8111-111111111111", + "backendName" : "pool-001-i-0007", + "url" : "https://i-0007.pool.invalid:8443", + "externalUrl" : "https://i-0007.external.invalid:8443", + "phase" : "ACTIVE", + "generation" : 4, + "controllerEpoch" : 7, + "podUid" : "pod-uid-0007", + "bootId" : "boot-0007", + "nodeId" : "node-0007", + "coordinatorId" : "abcde", + "configRevision" : "r-42", + "certifiedRevision" : "r-42", + "authRevision" : "auth-9", + "repair" : false, + "repairFor" : null, + "retirementKind" : null, + "pendingRequests" : 0, + "openTransactions" : 0, + "activeQueries" : 0, + "readyToSeal" : false, + "drained" : false, + "eligible" : true, + "membershipGeneration" : 19, + "replayed" : false +} ] diff --git a/controlplane/trinogateway/testdata/obligations.json b/controlplane/trinogateway/testdata/obligations.json new file mode 100644 index 000000000..b81b29b2e --- /dev/null +++ b/controlplane/trinogateway/testdata/obligations.json @@ -0,0 +1,12 @@ +{ + "protocolVersion" : 1, + "instanceId" : "i-0007", + "incarnation" : "11111111-1111-4111-8111-111111111111", + "phase" : "DRAINING", + "generation" : 5, + "pendingRequests" : 2, + "openTransactions" : 1, + "activeQueries" : 3, + "readyToSeal" : false, + "drained" : false +} diff --git a/controlplane/trinogateway/testdata/operation_history.json b/controlplane/trinogateway/testdata/operation_history.json new file mode 100644 index 000000000..dccd7c6b0 --- /dev/null +++ b/controlplane/trinogateway/testdata/operation_history.json @@ -0,0 +1,14 @@ +{ + "protocolVersion" : 1, + "operationId" : "op-1", + "steps" : [ { + "stepId" : "admit", + "payloadHash" : "0000000000000000000000000000000000000000000000000000000000000000", + "controllerEpoch" : 7, + "outcome" : "OK", + "recordedAt" : "2026-09-18T00:00:00Z", + "result" : { + "phase" : "ACTIVE" + } + } ] +} diff --git a/controlplane/trinogateway/testdata/pool_state.json b/controlplane/trinogateway/testdata/pool_state.json new file mode 100644 index 000000000..d4fcb06a8 --- /dev/null +++ b/controlplane/trinogateway/testdata/pool_state.json @@ -0,0 +1,27 @@ +{ + "protocolVersion" : 1, + "poolId" : "pool-001", + "apiMode" : "POOLED", + "controllerEpoch" : 7, + "membershipGeneration" : 19, + "minServing" : 3, + "desiredMembers" : 3, + "maxSurge" : 1, + "maxRepair" : 1, + "desiredRevision" : "r-42", + "admittedRevision" : "r-41", + "tenantAdmissionEnabled" : true, + "counts" : { + "PREPARING" : 1, + "RETIRED" : 4, + "ACTIVE" : 3, + "LOST" : 1 + }, + "servingMembers" : 3, + "liveMembers" : 4, + "surgeInUse" : 1, + "repairInUse" : 0, + "openPublications" : 0, + "blocked" : [ ], + "replayed" : false +} diff --git a/controlplane/trinogateway/testdata/publication.json b/controlplane/trinogateway/testdata/publication.json new file mode 100644 index 000000000..9f5ea2a33 --- /dev/null +++ b/controlplane/trinogateway/testdata/publication.json @@ -0,0 +1,22 @@ +{ + "protocolVersion" : 1, + "publicationId" : "pub-1", + "poolId" : "pool-001", + "tenant" : "tenant-a", + "targetRevision" : "r-42", + "membershipGeneration" : 19, + "phase" : "OPEN", + "requiredMembers" : [ "i-0007" ], + "receipts" : [ { + "instanceId" : "i-0007", + "incarnation" : "11111111-1111-4111-8111-111111111111", + "podUid" : "pod-uid-0007", + "bootId" : "boot-0007", + "appliedRevision" : "r-42", + "authFingerprint" : "fingerprint-1" + } ], + "missingMembers" : [ ], + "tenantState" : "PENDING", + "admittedRevision" : null, + "replayed" : false +} diff --git a/controlplane/trinogateway/testdata/tenant_admission.json b/controlplane/trinogateway/testdata/tenant_admission.json new file mode 100644 index 000000000..fb3efc00e --- /dev/null +++ b/controlplane/trinogateway/testdata/tenant_admission.json @@ -0,0 +1,13 @@ +{ + "protocolVersion" : 1, + "poolId" : "pool-001", + "tenant" : "tenant-a", + "state" : "ADMITTED", + "admittedRevision" : "r-42", + "publicationId" : "pub-1", + "principalRevision" : "binding-1", + "principalCount" : 2, + "principalsHash" : "0000000000000000000000000000000000000000000000000000000000000000", + "principals" : [ "warehouse-one", "warehouse-one.alice" ], + "replayed" : false +} diff --git a/controlplane/trinogateway/types.go b/controlplane/trinogateway/types.go new file mode 100644 index 000000000..98529cb54 --- /dev/null +++ b/controlplane/trinogateway/types.go @@ -0,0 +1,366 @@ +package trinogateway + +// Wire types for the Gateway pooled-member protocol v1. +// +// These mirror the Java records in PoolStore (PoolState, Member, Obligations, +// Publication, TenantAdmission, FailureReceipt, OperationHistory) and the field +// names PoolLifecycleService actually reads out of each request body. They are +// pinned by decoding fixtures serialized by the real Java records, not by a +// hand-written copy of a design document. +// +// Two details that are easy to get wrong and fail only at runtime: +// +// - The Gateway computes each mutation's payload hash ITSELF, as a SHA-256 +// over the canonicalized request body. A client must not send a payloadHash +// field; the only exception is openPublication, where payloadHash is an +// explicit publication input. +// - Admission is ONE call, POST .../admit, carrying a nested receipt. There +// is no separate certificate or activate route. + +// Checks duckgres performs against a candidate. The Gateway records the list +// verbatim and does not claim to have performed any of them; it independently +// verifies the live process identity from the member's own endpoint. +const ( + CheckImage = "image" + CheckWorkers = "workers" + CheckCatalogRevision = "catalog-revision" + CheckAuthRevision = "auth-revision" + CheckOperationalConnection = "operational-connection" +) + +// Step is the idempotency envelope every mutation carries. The Gateway derives +// the payload hash from the whole body, so replaying an identical body resolves +// to the recorded result and a changed body under the same step is a conflict. +type Step struct { + OperationID string `json:"operationId"` + StepID string `json:"stepId"` + ControllerEpoch int64 `json:"controllerEpoch"` + // OwnerIdentity names the controller PROCESS holding the pool. The Gateway + // records it (`owner_identity = coalesce(:owner, owner_identity)`) and + // refuses an equal epoch presented by a different owner, so leaving it out + // left the recorded owner NULL and reduced the fence to the epoch alone - + // which admits an equal epoch from any other controller. It is part of the + // authority envelope, which the Gateway strips before it hashes the + // payload, so sending it cannot turn a replay into a conflict. + OwnerIdentity string `json:"ownerIdentity,omitempty"` +} + +// PoolState is PoolStore.PoolState. +type PoolState struct { + ProtocolVersion int `json:"protocolVersion"` + PoolID string `json:"poolId"` + APIMode string `json:"apiMode"` + ControllerEpoch int64 `json:"controllerEpoch"` + MembershipGeneration int64 `json:"membershipGeneration"` + MinServing int `json:"minServing"` + DesiredMembers int `json:"desiredMembers"` + MaxSurge int `json:"maxSurge"` + MaxRepair int `json:"maxRepair"` + DesiredRevision string `json:"desiredRevision"` + AdmittedRevision string `json:"admittedRevision"` + TenantAdmissionEnabled bool `json:"tenantAdmissionEnabled"` + Counts map[string]int64 `json:"counts"` + ServingMembers int64 `json:"servingMembers"` + LiveMembers int64 `json:"liveMembers"` + SurgeInUse int64 `json:"surgeInUse"` + RepairInUse int64 `json:"repairInUse"` + OpenPublications int64 `json:"openPublications"` + Blocked []string `json:"blocked"` + Replayed bool `json:"replayed"` +} + +// ConfigurePoolRequest is the PUT body. desiredMembers defaults to minServing +// on the Gateway side when absent, so it is always sent explicitly here. +type ConfigurePoolRequest struct { + Step + APIMode string `json:"apiMode"` + MinServing int `json:"minServing"` + DesiredMembers int `json:"desiredMembers"` + MaxSurge int `json:"maxSurge"` + MaxRepair int `json:"maxRepair"` + DesiredRevision string `json:"desiredRevision,omitempty"` + TenantAdmissionEnabled bool `json:"tenantAdmissionEnabled"` +} + +// Member is PoolStore.Member. +type Member struct { + ProtocolVersion int `json:"protocolVersion"` + PoolID string `json:"poolId"` + InstanceID string `json:"instanceId"` + Incarnation string `json:"incarnation"` + BackendName string `json:"backendName"` + URL string `json:"url"` + ExternalURL string `json:"externalUrl"` + Phase string `json:"phase"` + Generation int64 `json:"generation"` + ControllerEpoch int64 `json:"controllerEpoch"` + PodUID string `json:"podUid"` + BootID string `json:"bootId"` + NodeID string `json:"nodeId"` + CoordinatorID string `json:"coordinatorId"` + ConfigRevision string `json:"configRevision"` + CertifiedRevision string `json:"certifiedRevision"` + AuthRevision string `json:"authRevision"` + Repair bool `json:"repair"` + RepairFor string `json:"repairFor"` + RetirementKind string `json:"retirementKind"` + PendingRequests int64 `json:"pendingRequests"` + OpenTransactions int64 `json:"openTransactions"` + ActiveQueries int64 `json:"activeQueries"` + ReadyToSeal bool `json:"readyToSeal"` + Drained bool `json:"drained"` + Eligible bool `json:"eligible"` + MembershipGeneration int64 `json:"membershipGeneration"` + Replayed bool `json:"replayed"` +} + +// Obligations is PoolStore.Obligations, returned by its own endpoint. +// +// Drain completion is read from HERE, never from a Member response: Go decodes +// an absent JSON field as zero, so a Member that happens to omit the counters +// would look like a safely drained member. This record always carries them. +type Obligations struct { + ProtocolVersion int `json:"protocolVersion"` + InstanceID string `json:"instanceId"` + Incarnation string `json:"incarnation"` + Phase string `json:"phase"` + Generation int64 `json:"generation"` + PendingRequests int64 `json:"pendingRequests"` + OpenTransactions int64 `json:"openTransactions"` + ActiveQueries int64 `json:"activeQueries"` + ReadyToSeal bool `json:"readyToSeal"` + Drained bool `json:"drained"` +} + +// Outstanding reports the work still pinned to the member. +func (o Obligations) Outstanding() int64 { + return o.PendingRequests + o.OpenTransactions + o.ActiveQueries +} + +// RegisterMemberRequest creates an unroutable PREPARING member. +// +// backendName must already exist as a Gateway backend registration in this +// routing group: the Gateway takes the endpoint from that record rather than +// trusting a caller-supplied URL, and observes the coordinator's process +// identity itself. url is optional and, when sent, must match exactly. +type RegisterMemberRequest struct { + Step + InstanceID string `json:"instanceId"` + BackendName string `json:"backendName"` + URL string `json:"url,omitempty"` + PodUID string `json:"podUid"` + BootID string `json:"bootId"` + ConfigRevision string `json:"configRevision"` + // RepairFor charges the member to the repair budget instead of the single + // planned surge, and names the failed instance it replaces. + RepairFor string `json:"repairFor,omitempty"` +} + +// ValidationReceipt is the nested receipt of an admission. Every string field is +// required by the Gateway: an empty value is rejected as a validation error. +type ValidationReceipt struct { + CertificateHash string `json:"certificateHash"` + ConfigRevision string `json:"configRevision"` + AuthRevision string `json:"authRevision"` + PodUID string `json:"podUid"` + BootID string `json:"bootId"` + NodeID string `json:"nodeId"` + CoordinatorID string `json:"coordinatorId"` + ReadyWorkers int `json:"readyWorkers"` + Checks []string `json:"checks"` +} + +// AdmitMemberRequest is the single certified-activation call. +type AdmitMemberRequest struct { + Step + ExpectedGeneration int64 `json:"expectedGeneration"` + Receipt ValidationReceipt `json:"receipt"` +} + +// MemberStepRequest is the envelope for drain, seal, retire and retired. +type MemberStepRequest struct { + Step + ExpectedGeneration int64 `json:"expectedGeneration"` + // ResourcesAbsent is the operator's assertion on `retired`. The Gateway + // records it and never infers resource deletion for itself. + ResourcesAbsent bool `json:"resourcesAbsent,omitempty"` +} + +// SuspectMemberRequest excludes a member from new admissions. The reason is +// required and is recorded. +type SuspectMemberRequest struct { + Step + ExpectedGeneration int64 `json:"expectedGeneration"` + Reason string `json:"reason"` +} + +// LostMemberRequest claims a member's process terminated. Evidence is +// mandatory: a probe timeout is not death, and a partitioned but possibly live +// process needs an explicit destructive authorization instead. +type LostMemberRequest struct { + Step + ExpectedGeneration int64 `json:"expectedGeneration"` + Evidence string `json:"evidence"` + Termination TerminationProof `json:"termination"` + DestructiveAuthorization bool `json:"destructiveAuthorization,omitempty"` + Reason string `json:"reason,omitempty"` +} + +// TerminationProof binds a loss claim to one exact incarnation. +type TerminationProof struct { + PodUID string `json:"podUid"` + BootID string `json:"bootId"` + NodeID string `json:"nodeId"` + CoordinatorID string `json:"coordinatorId"` + Source string `json:"source"` + ObservedAt string `json:"observedAt,omitempty"` +} + +// Evidence values for a loss claim. +const ( + EvidenceProcessTerminated = "PROCESS_TERMINATED" + EvidenceDestructiveOverride = "DESTRUCTIVE_OVERRIDE" +) + +// FailureReceipt is PoolStore.FailureReceipt: a failed member's preserved +// obligations, never reported as a successful drain. +type FailureReceipt struct { + ProtocolVersion int `json:"protocolVersion"` + PoolID string `json:"poolId"` + InstanceID string `json:"instanceId"` + Incarnation string `json:"incarnation"` + Evidence string `json:"evidence"` + // Detail is the Gateway's free-form evidence record, kept as decoded JSON + // so a future field cannot silently change its meaning here. + Detail map[string]any `json:"detail"` + OutstandingAdmissions int64 `json:"outstandingAdmissions"` + OutstandingTransactions int64 `json:"outstandingTransactions"` + OutstandingQueries int64 `json:"outstandingQueries"` + RecordedAt string `json:"recordedAt"` +} + +// OperationStep is one recorded step outcome. +type OperationStep struct { + StepID string `json:"stepId"` + PayloadHash string `json:"payloadHash"` + ControllerEpoch int64 `json:"controllerEpoch"` + Outcome string `json:"outcome"` + RecordedAt string `json:"recordedAt"` + Result map[string]any `json:"result"` +} + +// OperationHistory is the read-back that resolves a lost response. +type OperationHistory struct { + ProtocolVersion int `json:"protocolVersion"` + OperationID string `json:"operationId"` + Steps []OperationStep `json:"steps"` +} + +// Step returns the recorded outcome of one step, if it was reached. +func (o OperationHistory) Step(stepID string) (OperationStep, bool) { + for _, step := range o.Steps { + if step.StepID == stepID { + return step, true + } + } + return OperationStep{}, false +} + +// OpenPublicationRequest opens the tenant publication barrier. payloadHash is +// an explicit input here, unlike the guard hash the Gateway computes itself. +type OpenPublicationRequest struct { + Step + PublicationID string `json:"publicationId"` + Tenant string `json:"tenant"` + TargetRevision string `json:"targetRevision"` + ExpectedMembershipGeneration int64 `json:"expectedMembershipGeneration"` + PayloadHash string `json:"payloadHash"` +} + +// PublicationReceiptRequest records one member's application-loaded state. +type PublicationReceiptRequest struct { + Step + InstanceID string `json:"instanceId"` + PodUID string `json:"podUid"` + BootID string `json:"bootId"` + AppliedRevision string `json:"appliedRevision"` + AuthFingerprint string `json:"authFingerprint"` +} + +// CommitPublicationRequest closes the barrier and opens the tenant gate. +type CommitPublicationRequest struct { + Step + ExpectedMembershipGeneration int64 `json:"expectedMembershipGeneration"` +} + +// PublicationReceipt is one recorded member acknowledgement. +type PublicationReceipt struct { + InstanceID string `json:"instanceId"` + Incarnation string `json:"incarnation"` + PodUID string `json:"podUid"` + BootID string `json:"bootId"` + AppliedRevision string `json:"appliedRevision"` + AuthFingerprint string `json:"authFingerprint"` +} + +// Publication is PoolStore.Publication. +type Publication struct { + ProtocolVersion int `json:"protocolVersion"` + PublicationID string `json:"publicationId"` + PoolID string `json:"poolId"` + Tenant string `json:"tenant"` + TargetRevision string `json:"targetRevision"` + MembershipGeneration int64 `json:"membershipGeneration"` + Phase string `json:"phase"` + RequiredMembers []string `json:"requiredMembers"` + Receipts []PublicationReceipt `json:"receipts"` + MissingMembers []string `json:"missingMembers"` + TenantState string `json:"tenantState"` + AdmittedRevision string `json:"admittedRevision"` + Replayed bool `json:"replayed"` +} + +// TenantAdmission is PoolStore.TenantAdmission. +type TenantAdmission struct { + ProtocolVersion int `json:"protocolVersion"` + PoolID string `json:"poolId"` + Tenant string `json:"tenant"` + State string `json:"state"` + AdmittedRevision string `json:"admittedRevision"` + PublicationID string `json:"publicationId"` + // PrincipalRevision, PrincipalCount and PrincipalsHash are the + // authoritative binding the Gateway's admission restriction keys on. + // + // Principals themselves are echoed only by the READ path. A write result is + // recorded as an idempotent step, so it stays bounded however many logins a + // tenant has: a publication response carries the count and the hash instead + // and leaves the list empty. + PrincipalRevision string `json:"principalRevision"` + PrincipalCount int `json:"principalCount"` + PrincipalsHash string `json:"principalsHash"` + Principals []string `json:"principals"` + Replayed bool `json:"replayed"` +} + +// PublishPrincipalsRequest publishes a tenant's authoritative principal set. +// +// The Gateway cannot derive these strings. A tenant's logins are one flat +// namespace produced by the controller's own projection - a root login that +// carries no separator at all, plus qualified per-user names - and they are not +// a function of the tenant identifier. Deriving them from the shape of a name +// would refuse legitimate root logins and could bind a principal to the wrong +// tenant, so the controller states them. +// +// The set is replaced whole: a login removed here stops being admitted. +type PublishPrincipalsRequest struct { + Step + Revision string `json:"revision"` + Principals []string `json:"principals"` +} + +// RevokeTenantRequest closes a tenant's admission gate. Revocation is not +// additive publication: it takes effect for new work immediately. +type RevokeTenantRequest struct { + Step + Reason string `json:"reason"` +} diff --git a/controlplane/trinogateway/wire_test.go b/controlplane/trinogateway/wire_test.go new file mode 100644 index 000000000..a5b50f0f2 --- /dev/null +++ b/controlplane/trinogateway/wire_test.go @@ -0,0 +1,150 @@ +package trinogateway + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// These fixtures are NOT hand-written. They are produced by serializing the +// Gateway's real PoolStore records with Jackson (see the fixture generator +// recorded in STATUS-duckgres.md) and copied here verbatim. Decoding them is +// what establishes that this client and the Java producer agree; a test that +// invents both sides of the wire establishes nothing. +func readFixture(t *testing.T, name string, target any) { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatalf("read fixture %s: %v", name, err) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + // Unknown fields are a contract drift signal: the Java side grew a field + // this client does not know about, and silently ignoring it is how a + // consumer ends up making decisions on a stale view. + decoder.DisallowUnknownFields() + if err := decoder.Decode(target); err != nil { + t.Fatalf("decode %s against the Go type: %v", name, err) + } +} + +func TestPoolStateFixtureDecodes(t *testing.T) { + var pool PoolState + readFixture(t, "pool_state.json", &pool) + + if pool.PoolID != "pool-001" || pool.APIMode != "POOLED" { + t.Fatalf("pool = %+v", pool) + } + // The Java record calls these desiredMembers and maxRepair. An earlier + // draft of this client used desiredCount/repairBudget, which decoded to + // zero and would have reported a pool with no desired members. + if pool.DesiredMembers != 3 { + t.Fatalf("desiredMembers = %d, want 3", pool.DesiredMembers) + } + if pool.MaxRepair != 1 { + t.Fatalf("maxRepair = %d, want 1", pool.MaxRepair) + } + if pool.ControllerEpoch != 7 || pool.MembershipGeneration != 19 { + t.Fatalf("epoch/generation = %d/%d", pool.ControllerEpoch, pool.MembershipGeneration) + } + if pool.Counts["ACTIVE"] != 3 || pool.Counts["LOST"] != 1 { + t.Fatalf("counts = %v", pool.Counts) + } +} + +func TestMemberFixtureDecodes(t *testing.T) { + var member Member + readFixture(t, "member.json", &member) + + if member.InstanceID != "i-0007" || member.Phase != "ACTIVE" || member.Generation != 4 { + t.Fatalf("member = %+v", member) + } + if member.URL == "" || member.ExternalURL == "" { + t.Fatal("the endpoint fields did not decode") + } + // The Gateway records an auth revision on the member; an earlier draft of + // this client asserted no such thing could exist. + if member.AuthRevision != "auth-9" { + t.Fatalf("authRevision = %q", member.AuthRevision) + } + if member.Incarnation != "11111111-1111-4111-8111-111111111111" { + t.Fatalf("incarnation = %q", member.Incarnation) + } +} + +// GET members returns a bare JSON array. An envelope-shaped client decodes it +// as an empty list, which reads as "this pool has no members" - the most +// dangerous possible misreading for a controller that creates capacity. +func TestMembersFixtureIsABareArray(t *testing.T) { + var members []Member + readFixture(t, "members.json", &members) + if len(members) != 1 || members[0].InstanceID != "i-0007" { + t.Fatalf("members = %+v", members) + } +} + +func TestObligationsFixtureDecodes(t *testing.T) { + var obligations Obligations + readFixture(t, "obligations.json", &obligations) + + if obligations.PendingRequests != 2 || obligations.OpenTransactions != 1 || obligations.ActiveQueries != 3 { + t.Fatalf("obligations = %+v", obligations) + } + if obligations.Outstanding() != 6 { + t.Fatalf("outstanding = %d, want 6", obligations.Outstanding()) + } + if obligations.Drained || obligations.ReadyToSeal { + t.Fatal("a member with outstanding work reported itself drained") + } +} + +func TestPublicationFixtureDecodes(t *testing.T) { + var publication Publication + readFixture(t, "publication.json", &publication) + + if publication.PublicationID != "pub-1" || publication.Phase != "OPEN" { + t.Fatalf("publication = %+v", publication) + } + if len(publication.Receipts) != 1 || publication.Receipts[0].AppliedRevision != "r-42" { + t.Fatalf("receipts = %+v", publication.Receipts) + } + if publication.TenantState != "PENDING" { + t.Fatalf("tenantState = %q", publication.TenantState) + } +} + +func TestTenantAdmissionFixtureDecodes(t *testing.T) { + var admission TenantAdmission + readFixture(t, "tenant_admission.json", &admission) + if admission.State != "ADMITTED" || admission.AdmittedRevision != "r-42" { + t.Fatalf("admission = %+v", admission) + } +} + +func TestFailureReceiptFixtureDecodes(t *testing.T) { + var receipt FailureReceipt + readFixture(t, "failure_receipt.json", &receipt) + + if receipt.Evidence != "PROCESS_TERMINATED" { + t.Fatalf("evidence = %q", receipt.Evidence) + } + // A failure receipt PRESERVES the work that was lost. Reporting it as zero + // would turn a crash into a clean drain in the operator's record. + if receipt.OutstandingTransactions != 1 || receipt.OutstandingQueries != 2 { + t.Fatalf("outstanding work was not preserved: %+v", receipt) + } +} + +func TestOperationHistoryFixtureDecodes(t *testing.T) { + var history OperationHistory + readFixture(t, "operation_history.json", &history) + + step, found := history.Step("admit") + if !found || step.Outcome != "OK" { + t.Fatalf("history = %+v", history) + } + if step.Result["phase"] != "ACTIVE" { + t.Fatalf("recorded result = %v", step.Result) + } +} diff --git a/controlplane/trinopool/blueprint.go b/controlplane/trinopool/blueprint.go new file mode 100644 index 000000000..a1ee8c886 --- /dev/null +++ b/controlplane/trinopool/blueprint.go @@ -0,0 +1,374 @@ +package trinopool + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "sort" + "strings" + "unicode" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +// A blueprint is the immutable, versioned rendering of the Golden Chart that +// Argo delivers and duckgres instantiates. It is deliberately NOT a template +// language: duckgres injects instance identity through env vars, labels, +// selectors and object names only, and never edits a rendered body. Everything +// else — JVM flags, probes, resources, volumes, sidecars — is the chart's, so +// an instantiated blueprint can be diffed against an ordinary chart render. +const ( + // SupportedBlueprintVersion is the only document version this build + // accepts. A newer document is refused rather than partially understood. + SupportedBlueprintVersion = 1 + + maxBlueprintBytes = 1 << 20 + maxConfigFileBytes = 256 << 10 + maxConfigFilesTotal = 768 << 10 // a ConfigMap caps at ~1MiB including keys + maxSharedResources = 32 + maxWorkerReplicas = 100 + maxIdentifierLength = 253 + configFileRoleCoord = "coordinator" + configFileRoleWorker = "worker" + requiredConfigFile = "config.properties" + requiredNodeFile = "node.properties" + blueprintDigestLength = 64 +) + +var ( + digestPinnedImage = regexp.MustCompile(`^[^\s@:]+(:[0-9]+)?/?[^\s@]*@sha256:[a-f0-9]{64}$`) + environmentVarName = regexp.MustCompile(`^[A-Z][A-Z0-9_]*$`) + configFileName = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$`) + releaseIdentifier = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._:+-]{0,127}$`) +) + +// Blueprint is the document Argo mounts for a shared-pool cell. +type Blueprint struct { + BlueprintVersion int `json:"blueprint_version"` + // Generation orders desired state across control planes. It must increase + // with every published release. + // + // It has to come from the config source, because nothing in the control + // plane can order releases on its own: any replica may win the pool + // authority, and each one knows only the files it has mounted. Holding the + // fence proves who may write, not that what they hold is current - so + // without an externally supplied ordinal, a replica carrying yesterday's + // blueprint could legitimately publish it over today's. + // + // Absent or zero is accepted and simply does not order anything, which is + // the behavior before any generator emits it. + Generation int64 `json:"generation,omitempty"` + ReleaseID string `json:"release_id"` + ChartVersion string `json:"chart_version"` + Image string `json:"image"` + Namespace string `json:"namespace"` + ServiceAccountName string `json:"service_account_name"` + Coordinator BlueprintWorkload `json:"coordinator"` + Worker BlueprintWorkload `json:"worker"` + ConfigFiles map[string]map[string]string `json:"config_files"` + SharedResources BlueprintSharedResources `json:"shared_resources"` + IdentityBinding BlueprintIdentityBinding `json:"identity_binding"` +} + +// BlueprintWorkload carries one validated PodTemplateSpec. Replicas is only +// meaningful for workers; a serving instance always has exactly one +// coordinator, which is why the coordinator has no replica field at all. +type BlueprintWorkload struct { + Replicas int32 `json:"replicas,omitempty"` + PodTemplate corev1.PodTemplateSpec `json:"pod_template"` +} + +// BlueprintSharedResources names objects the pool owns as a whole. Duckgres +// mounts and reads them, and every instance-scoped delete path excludes them: +// retiring an instance must never take the pool's auth Secret with it. +type BlueprintSharedResources struct { + Secrets []string `json:"secrets,omitempty"` + ConfigMaps []string `json:"config_maps,omitempty"` +} + +// BlueprintIdentityBinding is the whole contract for instance-specific values. +// Each field names where duckgres may write, so the set of mutations it can +// perform on a chart render is enumerable and reviewable. +type BlueprintIdentityBinding struct { + InstanceLabel string `json:"instance_label"` + ComponentLabel string `json:"component_label"` + DiscoveryURIEnv string `json:"discovery_uri_env"` + NodeEnvironmentEnv string `json:"node_environment_env"` + InstanceIDEnv string `json:"instance_id_env"` + CoordinatorHTTPPortName string `json:"coordinator_http_port_name"` + CoordinatorContainerName string `json:"coordinator_container_name"` + WorkerContainerName string `json:"worker_container_name"` +} + +// ParseBlueprint decodes and validates one blueprint document. Unknown fields +// and trailing documents are refused so a newer Argo render cannot be silently +// half-applied by an older binary. +func ParseBlueprint(data []byte) (*Blueprint, error) { + if len(data) == 0 || len(data) > maxBlueprintBytes { + return nil, errors.New("blueprint must be a non-empty document within the size limit") + } + var blueprint Blueprint + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&blueprint); err != nil { + return nil, fmt.Errorf("decode blueprint: %w", err) + } + if err := decoder.Decode(new(any)); !errors.Is(err, io.EOF) { + return nil, errors.New("blueprint must contain exactly one JSON document") + } + if err := blueprint.Validate(); err != nil { + return nil, err + } + return &blueprint, nil +} + +// Digest is the stable identity of the execution configuration. It is stored on +// every instance row and annotated onto every object the instance owns, so a +// stale leader's create is recognizable as a foreign or outdated object rather +// than adopted as a serving replacement. +func (b *Blueprint) Digest() string { + encoded, err := json.Marshal(b) + if err != nil { + // Only unencodable types could fail here, and every field is JSON + // data. Fall back to a digest that can never collide with a real one. + return strings.Repeat("0", blueprintDigestLength) + } + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +// Validate refuses anything that would make an instance non-reproducible, +// non-isolated, or dependent on values duckgres cannot control. +func (b *Blueprint) Validate() error { + if b.BlueprintVersion != SupportedBlueprintVersion { + return fmt.Errorf("unsupported blueprint version %d", b.BlueprintVersion) + } + if !releaseIdentifier.MatchString(b.ReleaseID) { + return errors.New("blueprint requires a release identifier") + } + if !releaseIdentifier.MatchString(b.ChartVersion) { + return errors.New("blueprint requires a chart version") + } + if b.Generation < 0 { + return errors.New("blueprint generation must not be negative") + } + // An unpinned image breaks the central promise of the model: the same spec + // digest must always mean the same bytes, on every pod start, forever. + if !digestPinnedImage.MatchString(b.Image) || len(b.Image) > 512 { + return errors.New("blueprint image must be pinned to a sha256 digest") + } + if len(validation.IsDNS1123Label(b.Namespace)) != 0 { + return errors.New("blueprint namespace must be a DNS label") + } + if len(validation.IsDNS1123Subdomain(b.ServiceAccountName)) != 0 { + return errors.New("blueprint service account must be a DNS subdomain") + } + if err := b.IdentityBinding.validate(); err != nil { + return err + } + if b.Worker.Replicas < 1 || b.Worker.Replicas > maxWorkerReplicas { + return fmt.Errorf("blueprint worker replicas must be between 1 and %d", maxWorkerReplicas) + } + if err := b.validateWorkload(b.Coordinator, b.IdentityBinding.CoordinatorContainerName); err != nil { + return fmt.Errorf("coordinator: %w", err) + } + if err := b.validateWorkload(b.Worker, b.IdentityBinding.WorkerContainerName); err != nil { + return fmt.Errorf("worker: %w", err) + } + if err := b.validateConfigFiles(); err != nil { + return err + } + return b.SharedResources.validate() +} + +func (i BlueprintIdentityBinding) validate() error { + labels := []string{i.InstanceLabel, i.ComponentLabel} + for _, label := range labels { + if len(validation.IsQualifiedName(label)) != 0 { + return fmt.Errorf("identity binding label %q is not a qualified name", label) + } + } + if labels[0] == labels[1] { + return errors.New("identity binding labels must differ") + } + names := map[string]bool{} + for _, name := range i.EnvNames() { + if !environmentVarName.MatchString(name) || len(name) > 128 { + return fmt.Errorf("identity binding %q is not an environment variable name", name) + } + if names[name] { + return fmt.Errorf("identity binding environment variable %q is bound twice", name) + } + names[name] = true + } + if len(validation.IsValidPortName(i.CoordinatorHTTPPortName)) != 0 { + return errors.New("identity binding coordinator port name is invalid") + } + containers := []string{i.CoordinatorContainerName, i.WorkerContainerName} + for _, name := range containers { + if len(validation.IsDNS1123Label(name)) != 0 { + return fmt.Errorf("identity binding container name %q is invalid", name) + } + } + return nil +} + +// EnvNames lists every environment variable duckgres injects. A pod template +// must leave all of them unset. +func (i BlueprintIdentityBinding) EnvNames() []string { + return []string{i.DiscoveryURIEnv, i.NodeEnvironmentEnv, i.InstanceIDEnv} +} + +func (b *Blueprint) validateWorkload(workload BlueprintWorkload, mainContainer string) error { + template := workload.PodTemplate + // The object's identity is duckgres's to assign; a template that names + // itself would make two instances collide on one name. + if template.Name != "" || template.GenerateName != "" || template.Namespace != "" { + return errors.New("pod template must not name or namespace itself") + } + if len(template.OwnerReferences) != 0 { + return errors.New("pod template must not declare owner references") + } + if _, pinned := template.Labels[b.IdentityBinding.InstanceLabel]; pinned { + return errors.New("pod template must not set the instance label") + } + if _, pinned := template.Labels[b.IdentityBinding.ComponentLabel]; pinned { + return errors.New("pod template must not set the component label") + } + spec := template.Spec + if spec.HostNetwork || spec.HostPID || spec.HostIPC { + return errors.New("pod template must not share host namespaces") + } + if spec.NodeName != "" { + return errors.New("pod template must not pin a node") + } + if spec.ServiceAccountName != "" && spec.ServiceAccountName != b.ServiceAccountName { + return errors.New("pod template service account must match the blueprint") + } + if len(spec.Containers) == 0 || len(spec.Containers) > 8 { + return errors.New("pod template must declare between one and eight containers") + } + + injected := map[string]bool{} + for _, name := range b.IdentityBinding.EnvNames() { + injected[name] = true + } + main := 0 + for _, container := range append(append([]corev1.Container{}, spec.Containers...), spec.InitContainers...) { + if container.Name == mainContainer { + main++ + if container.Image != b.Image { + return errors.New("main container image must be the blueprint release image") + } + } + // Every image, sidecars included, has to be pinned: an unpinned OPA or + // exporter tag silently changes what a "immutable" instance runs. + if !digestPinnedImage.MatchString(container.Image) { + return fmt.Errorf("container %q image is not pinned to a digest", container.Name) + } + for _, env := range container.Env { + if injected[env.Name] { + return fmt.Errorf("container %q already binds the injected variable %q", container.Name, env.Name) + } + } + } + if main != 1 { + return fmt.Errorf("pod template must declare exactly one %q container", mainContainer) + } + return nil +} + +func (b *Blueprint) validateConfigFiles() error { + roles := map[string]bool{configFileRoleCoord: true, configFileRoleWorker: true} + total := 0 + for role, files := range b.ConfigFiles { + if !roles[role] { + return fmt.Errorf("unknown config file role %q", role) + } + delete(roles, role) + for name, body := range files { + if !configFileName.MatchString(name) { + return fmt.Errorf("config file name %q is not a plain file name", name) + } + if len(body) > maxConfigFileBytes { + return fmt.Errorf("config file %q exceeds the per-file limit", name) + } + if strings.IndexFunc(body, func(r rune) bool { return r != '\n' && r != '\t' && r != '\r' && unicode.IsControl(r) }) != -1 { + return fmt.Errorf("config file %q contains control characters", name) + } + total += len(name) + len(body) + } + for _, required := range []string{requiredConfigFile, requiredNodeFile} { + if _, present := files[required]; !present { + return fmt.Errorf("config files for %q must include %s", role, required) + } + } + // Without these references the instance would inherit whatever + // discovery URI and node environment the chart baked in - i.e. it + // would join another instance's cluster. + if !strings.Contains(files[requiredConfigFile], b.IdentityBinding.envReference(b.IdentityBinding.DiscoveryURIEnv)) { + return fmt.Errorf("%s for %q must bind the discovery URI to %s", requiredConfigFile, role, b.IdentityBinding.DiscoveryURIEnv) + } + if !strings.Contains(files[requiredNodeFile], b.IdentityBinding.envReference(b.IdentityBinding.NodeEnvironmentEnv)) { + return fmt.Errorf("%s for %q must bind the node environment to %s", requiredNodeFile, role, b.IdentityBinding.NodeEnvironmentEnv) + } + } + if len(roles) != 0 { + missing := make([]string, 0, len(roles)) + for role := range roles { + missing = append(missing, role) + } + sort.Strings(missing) + return fmt.Errorf("blueprint is missing config files for %s", strings.Join(missing, ", ")) + } + if total > maxConfigFilesTotal { + return errors.New("config files exceed the ConfigMap budget") + } + return nil +} + +func (i BlueprintIdentityBinding) envReference(name string) string { + return "${ENV:" + name + "}" +} + +func (s BlueprintSharedResources) validate() error { + if len(s.Secrets)+len(s.ConfigMaps) > maxSharedResources { + return errors.New("blueprint declares too many shared resources") + } + for _, name := range append(append([]string{}, s.Secrets...), s.ConfigMaps...) { + if len(validation.IsDNS1123Subdomain(name)) != 0 || len(name) > maxIdentifierLength { + return fmt.Errorf("shared resource %q is not a valid object name", name) + } + } + return nil +} + +// Protects reports whether the named object belongs to the pool rather than to +// a single instance. Every instance-scoped delete consults this. +func (s BlueprintSharedResources) Protects(name string) bool { + for _, shared := range append(append([]string{}, s.Secrets...), s.ConfigMaps...) { + if shared == name { + return true + } + } + return false +} + +// MarshalSnapshot returns the blueprint as a JSON document for durable storage +// alongside an instance. The instance keeps its own copy because Argo may +// replace or prune the source ConfigMap for a new release, and a PREPARING, +// SERVING or DRAINING instance must keep running the configuration it was +// created with. +func (b *Blueprint) MarshalSnapshot() (string, error) { + encoded, err := json.Marshal(b) + if err != nil { + return "", fmt.Errorf("encode blueprint snapshot: %w", err) + } + return string(encoded), nil +} diff --git a/controlplane/trinopool/blueprint_test.go b/controlplane/trinopool/blueprint_test.go new file mode 100644 index 000000000..0d22a861b --- /dev/null +++ b/controlplane/trinopool/blueprint_test.go @@ -0,0 +1,245 @@ +package trinopool + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +var testImage = "registry.invalid/trino@sha256:" + strings.Repeat("ab", 32) + +func validBlueprint() *Blueprint { + return &Blueprint{ + BlueprintVersion: 1, + ReleaseID: "2026-09-18.1", + ChartVersion: "trino-0.1.0", + Image: testImage, + Namespace: "trino-cell-a", + ServiceAccountName: "trino", + Coordinator: BlueprintWorkload{PodTemplate: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "trino-coordinator", Image: testImage}}, + }}}, + Worker: BlueprintWorkload{Replicas: 4, PodTemplate: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "trino-worker", Image: testImage}}, + }}}, + ConfigFiles: map[string]map[string]string{ + "coordinator": { + "config.properties": "coordinator=true\ndiscovery.uri=${ENV:TRINO_DISCOVERY_URI}\n", + "node.properties": "node.environment=${ENV:TRINO_NODE_ENVIRONMENT}\n", + }, + "worker": { + "config.properties": "coordinator=false\ndiscovery.uri=${ENV:TRINO_DISCOVERY_URI}\n", + "node.properties": "node.environment=${ENV:TRINO_NODE_ENVIRONMENT}\n", + }, + }, + SharedResources: BlueprintSharedResources{ + Secrets: []string{"trino-auth", "trino-tenant-secrets"}, + ConfigMaps: []string{"trino-resource-groups"}, + }, + IdentityBinding: BlueprintIdentityBinding{ + InstanceLabel: "posthog.com/trino-instance", + ComponentLabel: "app.kubernetes.io/component", + DiscoveryURIEnv: "TRINO_DISCOVERY_URI", + NodeEnvironmentEnv: "TRINO_NODE_ENVIRONMENT", + InstanceIDEnv: "DUCKGRES_TRINO_INSTANCE_ID", + CoordinatorHTTPPortName: "https", + CoordinatorContainerName: "trino-coordinator", + WorkerContainerName: "trino-worker", + }, + } +} + +func TestBlueprintValidateAcceptsAWellFormedDocument(t *testing.T) { + if err := validBlueprint().Validate(); err != nil { + t.Fatalf("valid blueprint rejected: %v", err) + } +} + +func TestBlueprintValidateRejects(t *testing.T) { + cases := map[string]func(*Blueprint){ + // An unpinned image makes the "immutable instance" claim false: the same + // spec digest could resolve to different bytes on every pod start. + "tag-only image": func(b *Blueprint) { + b.Image = "registry.invalid/trino:latest" + b.Coordinator.PodTemplate.Spec.Containers[0].Image = b.Image + b.Worker.PodTemplate.Spec.Containers[0].Image = b.Image + }, + "coordinator image differs from the pinned release": func(b *Blueprint) { + b.Coordinator.PodTemplate.Spec.Containers[0].Image = "registry.invalid/other@sha256:" + strings.Repeat("cd", 32) + }, + "unpinned sidecar image": func(b *Blueprint) { + b.Coordinator.PodTemplate.Spec.Containers = append(b.Coordinator.PodTemplate.Spec.Containers, + corev1.Container{Name: "opa", Image: "openpolicyagent/opa:latest"}) + }, + "missing coordinator container": func(b *Blueprint) { + b.Coordinator.PodTemplate.Spec.Containers[0].Name = "something-else" + }, + // Duckgres owns these env vars; a template that already sets one would + // silently win or lose depending on ordering. + "template already binds the identity env": func(b *Blueprint) { + b.Coordinator.PodTemplate.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "TRINO_DISCOVERY_URI", Value: "https://elsewhere.invalid"}} + }, + "template already sets the instance label": func(b *Blueprint) { + b.Worker.PodTemplate.Labels = map[string]string{"posthog.com/trino-instance": "pinned"} + }, + "template pins a node": func(b *Blueprint) { b.Worker.PodTemplate.Spec.NodeName = "ip-10-0-0-1" }, + "template uses host networking": func(b *Blueprint) { + b.Coordinator.PodTemplate.Spec.HostNetwork = true + }, + "template names the object": func(b *Blueprint) { b.Coordinator.PodTemplate.Name = "trino-coordinator" }, + "template pins a namespace": func(b *Blueprint) { b.Worker.PodTemplate.Namespace = "elsewhere" }, + // Without the ${ENV:...} reference the instance would inherit whatever + // discovery URI the chart baked in - i.e. another instance's coordinator. + "coordinator config does not bind the discovery URI": func(b *Blueprint) { + b.ConfigFiles["coordinator"]["config.properties"] = "coordinator=true\ndiscovery.uri=https://baked-in.invalid\n" + }, + "worker config does not bind the discovery URI": func(b *Blueprint) { + b.ConfigFiles["worker"]["config.properties"] = "coordinator=false\n" + }, + "missing node.properties": func(b *Blueprint) { delete(b.ConfigFiles["coordinator"], "node.properties") }, + "unknown config file role": func(b *Blueprint) { + b.ConfigFiles["sidecar"] = map[string]string{"x.properties": "a=b"} + }, + "config file name escapes the mount": func(b *Blueprint) { + b.ConfigFiles["worker"]["../evil.properties"] = "a=b" + }, + "config payload exceeds the ConfigMap budget": func(b *Blueprint) { + b.ConfigFiles["worker"]["big.properties"] = strings.Repeat("x", 1<<20) + }, + "zero workers": func(b *Blueprint) { b.Worker.Replicas = 0 }, + "invalid namespace": func(b *Blueprint) { b.Namespace = "Trino_Cell" }, + "unsupported blueprint version": func(b *Blueprint) { b.BlueprintVersion = 2 }, + "missing release id": func(b *Blueprint) { b.ReleaseID = "" }, + "identity env is not an environment variable name": func(b *Blueprint) { + b.IdentityBinding.DiscoveryURIEnv = "trino discovery" + }, + "identity envs collide": func(b *Blueprint) { + b.IdentityBinding.InstanceIDEnv = b.IdentityBinding.DiscoveryURIEnv + }, + "shared resource is not a valid object name": func(b *Blueprint) { + b.SharedResources.Secrets = []string{"Not A Secret"} + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + blueprint := validBlueprint() + mutate(blueprint) + if err := blueprint.Validate(); err == nil { + t.Fatalf("expected %s to be rejected", name) + } + }) + } +} + +func TestParseBlueprintRejectsUnknownFields(t *testing.T) { + data, err := json.Marshal(validBlueprint()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var document map[string]any + if err := json.Unmarshal(data, &document); err != nil { + t.Fatalf("unmarshal: %v", err) + } + document["future_field"] = true + extended, err := json.Marshal(document) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if _, err := ParseBlueprint(extended); err == nil { + t.Fatal("expected an unknown blueprint field to be rejected") + } +} + +func TestParseBlueprintRejectsTrailingDocuments(t *testing.T) { + data, err := json.Marshal(validBlueprint()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if _, err := ParseBlueprint(append(data, []byte("{}")...)); err == nil { + t.Fatal("expected a second JSON document to be rejected") + } +} + +func TestParseBlueprintRoundTrip(t *testing.T) { + data, err := json.Marshal(validBlueprint()) + if err != nil { + t.Fatalf("marshal: %v", err) + } + parsed, err := ParseBlueprint(data) + if err != nil { + t.Fatalf("parse: %v", err) + } + if parsed.Digest() != validBlueprint().Digest() { + t.Fatal("round-tripping the blueprint changed its digest") + } +} + +// The digest is what pins an instance to its exact execution configuration, so +// it has to be stable across process restarts and sensitive to every field. +func TestBlueprintDigestIsStableAndSensitive(t *testing.T) { + base := validBlueprint().Digest() + if base != validBlueprint().Digest() { + t.Fatal("digest is not deterministic") + } + if len(base) != 64 { + t.Fatalf("digest %q is not a sha256 hex string", base) + } + changed := validBlueprint() + changed.Worker.Replicas = 5 + if changed.Digest() == base { + t.Fatal("worker replica count did not change the digest") + } +} + +// The file a real deployment mounts has to parse and validate; a fixture that +// only exists in Go proves nothing about the document Argo delivers. +func TestParseBlueprintAcceptsTheCheckedInFixture(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", "blueprint.json")) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + blueprint, err := ParseBlueprint(data) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + if blueprint.ReleaseID == "" || blueprint.Worker.Replicas < 1 { + t.Fatal("fixture did not populate the release identity") + } +} + +// The generation orders desired state across control planes. Nothing in the +// control plane can order releases itself - any replica may win the pool +// authority and each knows only its own mounted files - so the value comes from +// the config source. Absent is accepted and orders nothing, which is the +// behavior before any generator emits it. +func TestBlueprintGenerationIsOptionalAndOrdered(t *testing.T) { + blueprint := validBlueprint() + if blueprint.Generation != 0 { + t.Fatal("the fixture already carries a generation") + } + if err := blueprint.Validate(); err != nil { + t.Fatalf("a blueprint without a generation was rejected: %v", err) + } + + blueprint.Generation = 7 + if err := blueprint.Validate(); err != nil { + t.Fatalf("a generation was rejected: %v", err) + } + blueprint.Generation = -1 + if err := blueprint.Validate(); err == nil { + t.Fatal("a negative generation was accepted") + } + + // It is part of the identity: a new generation is a new spec digest, so an + // instance created under one release is not mistaken for another. + first := validBlueprint() + second := validBlueprint() + second.Generation = 9 + if first.Digest() == second.Digest() { + t.Fatal("the generation did not change the blueprint digest") + } +} diff --git a/controlplane/trinopool/catalog_version.go b/controlplane/trinopool/catalog_version.go new file mode 100644 index 000000000..8cf6a4703 --- /dev/null +++ b/controlplane/trinopool/catalog_version.go @@ -0,0 +1,94 @@ +// Package trinopool holds the durable model and the pure logic of the shared +// Trino compute pool: the catalog publisher's wire-compatible hashing, the +// immutable instance blueprint, and the pool/instance/operation state machine. +// Nothing in this package talks to Kubernetes, so it builds and is tested +// without the `kubernetes` build tag. +package trinopool + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "sort" + "unicode/utf16" +) + +// CatalogVersion reproduces, byte for byte, the catalog version that the Trino +// coordinator computes in +// io.trino.plugin.catalogstore.posthog.CatalogVersions#computeCatalogVersion. +// +// Duckgres writes `trino_catalogs` rows directly, so a version that differs by +// a single byte makes every coordinator treat an unchanged catalog as a new +// one. Three details of the Java code are load-bearing and easy to get wrong in +// Go: +// +// - Guava's Hasher.putUnencodedChars writes UTF-16 code units little-endian, +// not UTF-8 bytes. +// - The length prefix is Java's String.length(), the UTF-16 code-unit count, +// not the rune count and not the byte count. +// - ImmutableSortedMap orders keys with String.compareTo, i.e. by UTF-16 code +// unit. That differs from Go's UTF-8 byte order for supplementary-plane +// characters, which sort below U+E000 in UTF-16 and above it in UTF-8. +// +// Pinned by TestCatalogVersionMatchesJavaGoldenVector against the vector the +// Java test pins. Changing any of it invalidates every version already stored. +func CatalogVersion(catalogName, connectorName string, properties map[string]string) string { + digest := sha256.New() + hashChars(digest, "catalog-hash") + hashLengthPrefixed(digest, catalogName) + hashLengthPrefixed(digest, connectorName) + hashInt(digest, int32(len(properties))) + for _, key := range sortedUTF16Keys(properties) { + hashLengthPrefixed(digest, key) + hashLengthPrefixed(digest, properties[key]) + } + return hex.EncodeToString(digest.Sum(nil)) +} + +// sortedUTF16Keys orders keys the way Java's String.compareTo does. +func sortedUTF16Keys(properties map[string]string) []string { + keys := make([]string, 0, len(properties)) + for key := range properties { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { return lessUTF16(keys[i], keys[j]) }) + return keys +} + +// lessUTF16 compares two strings by UTF-16 code unit, matching +// java.lang.String#compareTo. +func lessUTF16(left, right string) bool { + leftUnits, rightUnits := utf16.Encode([]rune(left)), utf16.Encode([]rune(right)) + for index := 0; index < len(leftUnits) && index < len(rightUnits); index++ { + if leftUnits[index] != rightUnits[index] { + return leftUnits[index] < rightUnits[index] + } + } + return len(leftUnits) < len(rightUnits) +} + +type byteWriter interface{ Write([]byte) (int, error) } + +func hashLengthPrefixed(digest byteWriter, value string) { + units := utf16.Encode([]rune(value)) + hashInt(digest, int32(len(units))) + hashUnits(digest, units) +} + +func hashChars(digest byteWriter, value string) { + hashUnits(digest, utf16.Encode([]rune(value))) +} + +func hashUnits(digest byteWriter, units []uint16) { + buffer := make([]byte, 2*len(units)) + for index, unit := range units { + binary.LittleEndian.PutUint16(buffer[2*index:], unit) + } + _, _ = digest.Write(buffer) +} + +func hashInt(digest byteWriter, value int32) { + buffer := make([]byte, 4) + binary.LittleEndian.PutUint32(buffer, uint32(value)) + _, _ = digest.Write(buffer) +} diff --git a/controlplane/trinopool/catalog_version_test.go b/controlplane/trinopool/catalog_version_test.go new file mode 100644 index 000000000..bef388b70 --- /dev/null +++ b/controlplane/trinopool/catalog_version_test.go @@ -0,0 +1,136 @@ +package trinopool + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "testing" + "unicode/utf16" +) + +// The Trino coordinator derives a catalog's version from its own +// io.trino.plugin.catalogstore.posthog.CatalogVersions implementation. Once +// duckgres writes catalog rows directly it has to produce byte-identical +// versions, otherwise every existing row looks changed to every coordinator. +// This vector is the one pinned by the Java test. +const javaGoldenCatalogVersion = "0b48ccd298e6f062b4f8d81466bbeab8ef9cea832c4c08356f7e661cad1d040c" + +func goldenProperties() map[string]string { + return map[string]string{ + "ducklake.metadata.connection-url": "jdbc:postgresql://db:5432/lake", + "ducklake.data-path": "s3://bucket/prefix/", + } +} + +func TestCatalogVersionMatchesJavaGoldenVector(t *testing.T) { + if version := CatalogVersion("orders", "ducklake", goldenProperties()); version != javaGoldenCatalogVersion { + t.Fatalf("catalog version = %q, want %q", version, javaGoldenCatalogVersion) + } +} + +func TestCatalogVersionIgnoresPropertyOrder(t *testing.T) { + first := CatalogVersion("orders", "ducklake", map[string]string{ + "ducklake.data-path": "s3://bucket/prefix/", + "ducklake.metadata.connection-url": "jdbc:postgresql://db:5432/lake", + }) + second := CatalogVersion("orders", "ducklake", goldenProperties()) + if first != second { + t.Fatalf("property order changed the version: %q vs %q", first, second) + } +} + +func TestCatalogVersionEveryInputChangesTheVersion(t *testing.T) { + base := CatalogVersion("orders", "ducklake", goldenProperties()) + cases := map[string]string{ + "catalog name": CatalogVersion("invoices", "ducklake", goldenProperties()), + "connector name": CatalogVersion("orders", "iceberg", goldenProperties()), + "extra property": CatalogVersion("orders", "ducklake", map[string]string{ + "ducklake.metadata.connection-url": "jdbc:postgresql://db:5432/lake", + "ducklake.data-path": "s3://bucket/prefix/", + "ducklake.max-split-size": "32MB", + }), + "changed value": CatalogVersion("orders", "ducklake", map[string]string{ + "ducklake.metadata.connection-url": "jdbc:postgresql://db:5432/lake", + "ducklake.data-path": "s3://other-bucket/prefix/", + }), + "no properties": CatalogVersion("orders", "ducklake", nil), + } + for name, version := range cases { + if version == base { + t.Errorf("%s did not change the catalog version", name) + } + } +} + +// Java hashes a length prefix before each string, so moving a character across +// the key/value boundary has to change the version. +func TestCatalogVersionLengthPrefixSeparatesAdjacentStrings(t *testing.T) { + if CatalogVersion("orders", "ducklake", map[string]string{"ab": "cd"}) == + CatalogVersion("orders", "ducklake", map[string]string{"a": "bcd"}) { + t.Fatal("adjacent strings are not separated by a length prefix") + } +} + +// Java sorts properties with String.compareTo, which compares UTF-16 code +// units. Go's natural string order is UTF-8 byte order, and the two disagree +// exactly when a supplementary-plane character meets one from U+E000..U+FFFF: +// the surrogate pair starts at 0xD83D in UTF-16, below U+E000, while its UTF-8 +// encoding starts at 0xF0, above U+E000's 0xEE. Sorting the Go way silently +// produces a version no coordinator agrees with. +func TestCatalogVersionSortsPropertiesInUTF16Order(t *testing.T) { + properties := map[string]string{ + "\U0001F600": "emoji", + "": "private-use", + } + want := referenceCatalogVersion("orders", "ducklake", [][2]string{ + {"\U0001F600", "emoji"}, + {"", "private-use"}, + }) + if got := CatalogVersion("orders", "ducklake", properties); got != want { + t.Fatalf("properties were not sorted in UTF-16 order: got %q, want %q", got, want) + } +} + +// Guards the reference implementation the previous test compares against: if it +// drifts from Java, its verdict is worthless. +func TestReferenceCatalogVersionMatchesJavaGoldenVector(t *testing.T) { + got := referenceCatalogVersion("orders", "ducklake", [][2]string{ + {"ducklake.data-path", "s3://bucket/prefix/"}, + {"ducklake.metadata.connection-url", "jdbc:postgresql://db:5432/lake"}, + }) + if got != javaGoldenCatalogVersion { + t.Fatalf("reference implementation = %q, want %q", got, javaGoldenCatalogVersion) + } +} + +// referenceCatalogVersion is a deliberately literal, independently written +// transcription of the Java hasher over already-ordered pairs. It exists so the +// ordering and length-prefix rules are pinned by something other than the +// implementation under test. +func referenceCatalogVersion(catalog, connector string, ordered [][2]string) string { + digest := sha256.New() + putChars := func(value string) { + for _, unit := range utf16.Encode([]rune(value)) { + _, _ = digest.Write([]byte{byte(unit), byte(unit >> 8)}) + } + } + putInt := func(value int32) { + buffer := make([]byte, 4) + binary.LittleEndian.PutUint32(buffer, uint32(value)) + _, _ = digest.Write(buffer) + } + putString := func(value string) { + putInt(int32(len(utf16.Encode([]rune(value))))) + putChars(value) + } + + putChars("catalog-hash") + putString(catalog) + putString(connector) + putInt(int32(len(ordered))) + for _, pair := range ordered { + putString(pair[0]) + putString(pair[1]) + } + return hex.EncodeToString(digest.Sum(nil)) +} diff --git a/controlplane/trinopool/instantiate.go b/controlplane/trinopool/instantiate.go new file mode 100644 index 000000000..4de3f8026 --- /dev/null +++ b/controlplane/trinopool/instantiate.go @@ -0,0 +1,315 @@ +package trinopool + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation" +) + +// Labels and annotations duckgres stamps on everything it owns. The epoch and +// spec digest are what make a stale leader's write recognizable: a conditional +// update compares them before touching an existing object, so a create that +// arrives late is at worst an unadmitted orphan, never a serving replacement. +const ( + LabelInstance = "posthog.com/trino-instance" + LabelPool = "posthog.com/trino-pool" + LabelManagedBy = "app.kubernetes.io/managed-by" + ManagedByValue = "duckgres-trino-pool" + + AnnotationSpecDigest = "posthog.com/trino-spec-digest" + AnnotationAuthorityEpoch = "posthog.com/trino-authority-epoch" + AnnotationReleaseID = "posthog.com/trino-release-id" + + componentCoordinator = "coordinator" + componentWorker = "worker" +) + +// Identity is everything instance-specific duckgres injects. It is a closed +// set on purpose: these fields, and nothing else, are what distinguishes one +// instantiated blueprint from another. +type Identity struct { + PoolID string + PoolLabelValue string + InstanceID string + NodeEnvironment string + AuthorityEpoch int64 + CoordinatorPort int32 + DiscoveryURIHost string +} + +func (i Identity) validate() error { + if len(validation.IsDNS1123Label(i.InstanceID)) != 0 { + return errors.New("instance identity must be a DNS label") + } + if len(validation.IsDNS1123Label(i.PoolLabelValue)) != 0 { + return errors.New("pool label value must be a DNS label") + } + if i.NodeEnvironment == "" || len(i.NodeEnvironment) > 128 { + return errors.New("instance identity requires a node environment") + } + if len(validation.IsDNS1123Subdomain(i.DiscoveryURIHost)) != 0 { + return errors.New("instance identity requires a discovery host") + } + if i.CoordinatorPort < 1 || i.CoordinatorPort > 65535 { + return errors.New("instance identity requires a valid coordinator port") + } + return nil +} + +// Objects is one instance's complete Kubernetes inventory. Pool-shared objects +// (namespace, service accounts, auth Secret, TLS, resource groups) are NOT here +// and are never created or deleted by the instance lifecycle. +type Objects struct { + ConfigMap *corev1.ConfigMap + WorkerConfigMap *corev1.ConfigMap + Service *corev1.Service + CoordinatorDeployment *appsv1.Deployment + WorkerDeployment *appsv1.Deployment +} + +// All returns every object, for uniform stamping and inventory checks. +func (o Objects) All() []metav1.Object { + return []metav1.Object{o.ConfigMap, o.WorkerConfigMap, o.Service, o.CoordinatorDeployment, o.WorkerDeployment} +} + +// SpecDigest identifies the exact execution configuration of one instance: the +// blueprint plus the identity injected into it. It is stored on the instance +// row and annotated on every object, so "is this object mine and current?" is a +// string comparison rather than a deep diff. +// +// The authority epoch is deliberately EXCLUDED. It is not execution +// configuration — it is who last wrote the object — and it has its own +// annotation that the ownership check compares numerically. Folding it in +// wedged an instance permanently: a leader that recorded an instance and +// created its objects, then died before the phase CAS, left the successor +// (one epoch higher) computing a different digest, so every later apply hit +// AlreadyExists and was refused as a foreign object, forever, with the +// instance still counted as preparing and the whole pool blocked behind it. +func (b *Blueprint) SpecDigest(identity Identity) string { + pinned := identity + pinned.AuthorityEpoch = 0 + encoded, err := json.Marshal(struct { + Blueprint string `json:"blueprint"` + Identity Identity `json:"identity"` + }{Blueprint: b.Digest(), Identity: pinned}) + if err != nil { + return "" + } + sum := sha256.Sum256(encoded) + return hex.EncodeToString(sum[:]) +} + +// Instantiate renders one instance's objects from the blueprint. +// +// This is deliberately not a template engine. The pod templates and config file +// bodies are carried through byte for byte; duckgres only sets object names, +// labels, selectors, replica counts and the three identity env vars the +// blueprint's identity binding declares. That is what lets an instantiated +// blueprint be diffed against an ordinary chart render in a test. +func (b *Blueprint) Instantiate(identity Identity) (Objects, error) { + if err := b.Validate(); err != nil { + return Objects{}, err + } + if err := identity.validate(); err != nil { + return Objects{}, err + } + + digest := b.SpecDigest(identity) + meta := func(name string) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Name: name, + Namespace: b.Namespace, + Labels: map[string]string{ + LabelInstance: identity.InstanceID, + LabelPool: identity.PoolLabelValue, + LabelManagedBy: ManagedByValue, + }, + Annotations: map[string]string{ + AnnotationSpecDigest: digest, + AnnotationAuthorityEpoch: strconv.FormatInt(identity.AuthorityEpoch, 10), + AnnotationReleaseID: b.ReleaseID, + }, + } + } + + coordinatorConfig := &corev1.ConfigMap{ + ObjectMeta: meta(identity.InstanceID + "-coordinator-config"), + Data: b.ConfigFiles[configFileRoleCoord], + } + workerConfig := &corev1.ConfigMap{ + ObjectMeta: meta(identity.InstanceID + "-worker-config"), + Data: b.ConfigFiles[configFileRoleWorker], + } + + // One Service per instance, selecting only that instance's coordinator. + // Load-balancing across coordinators would hand a worker to a foreign + // cluster and silently merge two instances. + service := &corev1.Service{ + ObjectMeta: meta(identity.InstanceID), + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + LabelInstance: identity.InstanceID, + b.IdentityBinding.ComponentLabel: componentCoordinator, + }, + Ports: []corev1.ServicePort{{ + Name: b.IdentityBinding.CoordinatorHTTPPortName, + Port: identity.CoordinatorPort, + TargetPort: intOrStringFromName(b.IdentityBinding.CoordinatorHTTPPortName), + Protocol: corev1.ProtocolTCP, + }}, + }, + } + + // Workers discover their coordinator over the cluster-internal HTTP port. + // TLS terminates at the Gateway, so there is no in-cluster certificate for + // a worker to verify and none is pretended. + discoveryURI := fmt.Sprintf("http://%s:%d", identity.DiscoveryURIHost, identity.CoordinatorPort) + coordinator := b.deployment(identity, meta(identity.InstanceID+"-coordinator"), componentCoordinator, + b.Coordinator, 1, coordinatorConfig.Name, discoveryURI) + worker := b.deployment(identity, meta(identity.InstanceID+"-worker"), componentWorker, + b.Worker, b.Worker.Replicas, workerConfig.Name, discoveryURI) + + return Objects{ + ConfigMap: coordinatorConfig, + WorkerConfigMap: workerConfig, + Service: service, + CoordinatorDeployment: coordinator, + WorkerDeployment: worker, + }, nil +} + +func (b *Blueprint) deployment( + identity Identity, + objectMeta metav1.ObjectMeta, + component string, + workload BlueprintWorkload, + replicas int32, + configMapName string, + discoveryURI string, +) *appsv1.Deployment { + template := *workload.PodTemplate.DeepCopy() + if template.Labels == nil { + template.Labels = map[string]string{} + } + template.Labels[LabelInstance] = identity.InstanceID + template.Labels[LabelPool] = identity.PoolLabelValue + template.Labels[b.IdentityBinding.ComponentLabel] = component + if template.Annotations == nil { + template.Annotations = map[string]string{} + } + // Roll the pods when the instance's own config changes. In practice an + // instance's config never changes — a new release is a new instance — so + // this only matters as a safety net. + template.Annotations[AnnotationSpecDigest] = objectMeta.Annotations[AnnotationSpecDigest] + template.Spec.ServiceAccountName = b.ServiceAccountName + + // Scope every pod anti-affinity term to THIS instance. + // + // The chart spreads workers across nodes with a selector that matches the + // whole Trino app. Left alone, that selector would also match the workers + // of every other instance in the pool, so a three-instance pool would + // fight itself for nodes and the surge instance might never schedule. The + // fix is to narrow the existing terms, not to drop them: spreading one + // instance's workers is still what we want. + scopeAntiAffinityToInstance(&template.Spec, identity.InstanceID) + + identityEnv := []corev1.EnvVar{ + {Name: b.IdentityBinding.DiscoveryURIEnv, Value: discoveryURI}, + {Name: b.IdentityBinding.NodeEnvironmentEnv, Value: identity.NodeEnvironment}, + {Name: b.IdentityBinding.InstanceIDEnv, Value: identity.InstanceID}, + } + for index := range template.Spec.Containers { + template.Spec.Containers[index].Env = append(template.Spec.Containers[index].Env, identityEnv...) + } + for index := range template.Spec.InitContainers { + template.Spec.InitContainers[index].Env = append(template.Spec.InitContainers[index].Env, identityEnv...) + } + // The config volume, if the chart declared one, is pointed at THIS + // instance's ConfigMap. Argo may replace the blueprint's source ConfigMap + // for a new release; a running instance must keep reading its own copy. + for index := range template.Spec.Volumes { + volume := &template.Spec.Volumes[index] + if volume.Name == "config" && volume.ConfigMap != nil { + volume.ConfigMap.Name = configMapName + } + } + if !hasVolume(template.Spec.Volumes, "config") { + template.Spec.Volumes = append(template.Spec.Volumes, corev1.Volume{ + Name: "config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: configMapName}, + }, + }, + }) + } + + strategy := appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} + count := replicas + return &appsv1.Deployment{ + ObjectMeta: objectMeta, + Spec: appsv1.DeploymentSpec{ + Replicas: &count, + // Recreate for both: a serving instance is never rolled in place. + // Replacement means a new instance with a new identity, which is + // what keeps "immutable instance" true. + Strategy: strategy, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{ + LabelInstance: identity.InstanceID, + b.IdentityBinding.ComponentLabel: component, + }}, + Template: template, + }, + } +} + +func hasVolume(volumes []corev1.Volume, name string) bool { + for _, volume := range volumes { + if volume.Name == name { + return true + } + } + return false +} + +// intOrStringFromName targets the container port by NAME, so the chart stays +// free to change the numeric port without duckgres rewriting its render. +func intOrStringFromName(name string) intstr.IntOrString { + return intstr.FromString(name) +} + +// scopeAntiAffinityToInstance narrows each anti-affinity term's label selector +// with the instance label, preserving the chart's topology keys and weights. +// Terms that already select on the instance label are left alone. +func scopeAntiAffinityToInstance(spec *corev1.PodSpec, instanceID string) { + if spec.Affinity == nil || spec.Affinity.PodAntiAffinity == nil { + return + } + antiAffinity := spec.Affinity.PodAntiAffinity + for index := range antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution { + scopeAffinityTerm(&antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[index], instanceID) + } + for index := range antiAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + scopeAffinityTerm(&antiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[index].PodAffinityTerm, instanceID) + } +} + +func scopeAffinityTerm(term *corev1.PodAffinityTerm, instanceID string) { + if term.LabelSelector == nil { + term.LabelSelector = &metav1.LabelSelector{} + } + if term.LabelSelector.MatchLabels == nil { + term.LabelSelector.MatchLabels = map[string]string{} + } + term.LabelSelector.MatchLabels[LabelInstance] = instanceID +} diff --git a/controlplane/trinopool/instantiate_test.go b/controlplane/trinopool/instantiate_test.go new file mode 100644 index 000000000..5acc83036 --- /dev/null +++ b/controlplane/trinopool/instantiate_test.go @@ -0,0 +1,301 @@ +package trinopool + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func testIdentity() Identity { + return Identity{ + PoolID: "registered:cell-001", + PoolLabelValue: "cell-001", + InstanceID: "cell-001-a1b2c3d4", + NodeEnvironment: "mw_dev_pool_001", + AuthorityEpoch: 7, + CoordinatorPort: 8443, + DiscoveryURIHost: "cell-001-a1b2c3d4.trino-cell-001.svc.cluster.local", + } +} + +func instantiate(t *testing.T) Objects { + t.Helper() + objects, err := validBlueprint().Instantiate(testIdentity()) + if err != nil { + t.Fatalf("instantiate: %v", err) + } + return objects +} + +func TestInstantiateNamesEverythingAfterTheInstance(t *testing.T) { + objects := instantiate(t) + cases := map[string]string{ + "config map": objects.ConfigMap.Name, + "service": objects.Service.Name, + "coordinator deployment": objects.CoordinatorDeployment.Name, + "worker deployment": objects.WorkerDeployment.Name, + } + for what, name := range cases { + if !strings.HasPrefix(name, "cell-001-a1b2c3d4") { + t.Errorf("%s is named %q, which is not scoped to the instance", what, name) + } + } + if objects.CoordinatorDeployment.Name == objects.WorkerDeployment.Name { + t.Fatal("coordinator and worker deployments share a name") + } +} + +// Workers must discover only their own coordinator. A shared Service or a +// discovery URI pointing at another instance would silently merge two clusters. +func TestInstantiateBindsWorkersToTheirOwnCoordinator(t *testing.T) { + objects := instantiate(t) + + selector := objects.Service.Spec.Selector + if selector["posthog.com/trino-instance"] != "cell-001-a1b2c3d4" { + t.Fatalf("service selector %v is not instance-scoped", selector) + } + if selector["app.kubernetes.io/component"] != "coordinator" { + t.Fatalf("service selector %v does not pin the coordinator", selector) + } + + discovery := "" + for _, env := range objects.WorkerDeployment.Spec.Template.Spec.Containers[0].Env { + if env.Name == "TRINO_DISCOVERY_URI" { + discovery = env.Value + } + } + if !strings.Contains(discovery, "cell-001-a1b2c3d4") { + t.Fatalf("worker discovery URI %q does not point at its own coordinator", discovery) + } +} + +// Every object carries the identity and the spec digest, so a stale leader's +// create is recognizable as foreign or outdated rather than adopted. +func TestInstantiateStampsOwnershipAndSpecDigest(t *testing.T) { + objects := instantiate(t) + digest := validBlueprint().SpecDigest(testIdentity()) + for _, object := range objects.All() { + labels := object.GetLabels() + if labels["posthog.com/trino-instance"] != "cell-001-a1b2c3d4" { + t.Errorf("%s is missing the instance label", object.GetName()) + } + if labels["posthog.com/trino-pool"] != "cell-001" { + t.Errorf("%s is missing the pool label", object.GetName()) + } + annotations := object.GetAnnotations() + if annotations[AnnotationSpecDigest] != digest { + t.Errorf("%s carries spec digest %q, want %q", object.GetName(), annotations[AnnotationSpecDigest], digest) + } + if annotations[AnnotationAuthorityEpoch] != "7" { + t.Errorf("%s carries authority epoch %q", object.GetName(), annotations[AnnotationAuthorityEpoch]) + } + if object.GetNamespace() != "trino-cell-a" { + t.Errorf("%s landed in namespace %q", object.GetName(), object.GetNamespace()) + } + } +} + +// The coordinator is a singleton: two coordinators behind one discovery URI +// would race. Recreate, never a rolling update inside a serving instance. +func TestInstantiateMakesTheCoordinatorASingleton(t *testing.T) { + objects := instantiate(t) + if objects.CoordinatorDeployment.Spec.Replicas == nil || *objects.CoordinatorDeployment.Spec.Replicas != 1 { + t.Fatal("coordinator is not a single replica") + } + if objects.CoordinatorDeployment.Spec.Strategy.Type != "Recreate" { + t.Fatalf("coordinator strategy = %q, want Recreate", objects.CoordinatorDeployment.Spec.Strategy.Type) + } + if objects.WorkerDeployment.Spec.Replicas == nil || *objects.WorkerDeployment.Spec.Replicas != 4 { + t.Fatal("worker replica count does not come from the blueprint") + } +} + +// The chart's own containers, volumes and probes must survive instantiation +// untouched: duckgres injects identity, it does not rewrite execution config. +func TestInstantiatePreservesTheChartRender(t *testing.T) { + blueprint := validBlueprint() + blueprint.Coordinator.PodTemplate.Spec.Containers = append(blueprint.Coordinator.PodTemplate.Spec.Containers, + corev1.Container{Name: "opa", Image: testImage}) + blueprint.Coordinator.PodTemplate.Spec.Containers[0].Args = []string{"--flag"} + + objects, err := blueprint.Instantiate(testIdentity()) + if err != nil { + t.Fatalf("instantiate: %v", err) + } + containers := objects.CoordinatorDeployment.Spec.Template.Spec.Containers + if len(containers) != 2 || containers[1].Name != "opa" { + t.Fatalf("sidecar was dropped: %+v", containers) + } + if len(containers[0].Args) != 1 || containers[0].Args[0] != "--flag" { + t.Fatal("chart-provided args were rewritten") + } + if containers[0].Image != blueprint.Image { + t.Fatal("container image was rewritten") + } +} + +// Identity env vars go on EVERY container of the pod, because a sidecar may +// consume them too, and the blueprint validator has already refused templates +// that set them. +func TestInstantiateInjectsIdentityIntoEveryContainer(t *testing.T) { + blueprint := validBlueprint() + blueprint.Coordinator.PodTemplate.Spec.Containers = append(blueprint.Coordinator.PodTemplate.Spec.Containers, + corev1.Container{Name: "opa", Image: testImage}) + objects, err := blueprint.Instantiate(testIdentity()) + if err != nil { + t.Fatalf("instantiate: %v", err) + } + for _, container := range objects.CoordinatorDeployment.Spec.Template.Spec.Containers { + found := map[string]bool{} + for _, env := range container.Env { + found[env.Name] = true + } + for _, name := range []string{"TRINO_DISCOVERY_URI", "TRINO_NODE_ENVIRONMENT", "DUCKGRES_TRINO_INSTANCE_ID"} { + if !found[name] { + t.Errorf("container %q is missing %s", container.Name, name) + } + } + } +} + +// Config files are mounted from the instance's OWN ConfigMap, so a new release +// cannot change what a running instance reads. +func TestInstantiateMountsTheInstanceConfigMap(t *testing.T) { + objects := instantiate(t) + if objects.ConfigMap.Data["config.properties"] == "" || objects.ConfigMap.Data["node.properties"] == "" { + t.Fatalf("config map is missing coordinator files: %v", objects.ConfigMap.Data) + } + if objects.WorkerConfigMap.Data["config.properties"] == "" { + t.Fatalf("worker config map is missing files: %v", objects.WorkerConfigMap.Data) + } + if objects.ConfigMap.Name == objects.WorkerConfigMap.Name { + t.Fatal("coordinator and worker share one config map") + } +} + +// The spec digest must change when anything that affects execution changes, +// and stay identical otherwise: it is what a conditional write compares. +func TestSpecDigestIsSensitiveToIdentityAndBlueprint(t *testing.T) { + blueprint := validBlueprint() + base := blueprint.SpecDigest(testIdentity()) + if base != blueprint.SpecDigest(testIdentity()) { + t.Fatal("spec digest is not deterministic") + } + + other := testIdentity() + other.InstanceID = "cell-001-ffffffff" + if blueprint.SpecDigest(other) == base { + t.Fatal("a different instance produced the same spec digest") + } + + changed := validBlueprint() + changed.Worker.Replicas = 5 + if changed.SpecDigest(testIdentity()) == base { + t.Fatal("a blueprint change produced the same spec digest") + } +} + +func TestInstantiateRejectsAnIncompleteIdentity(t *testing.T) { + cases := map[string]func(*Identity){ + "missing instance": func(i *Identity) { i.InstanceID = "" }, + "instance is not a label": func(i *Identity) { i.InstanceID = "Not_A_Label" }, + "missing discovery host": func(i *Identity) { i.DiscoveryURIHost = "" }, + "missing node env": func(i *Identity) { i.NodeEnvironment = "" }, + "missing pool label": func(i *Identity) { i.PoolLabelValue = "" }, + "invalid port": func(i *Identity) { i.CoordinatorPort = 0 }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + identity := testIdentity() + mutate(&identity) + if _, err := validBlueprint().Instantiate(identity); err == nil { + t.Fatalf("accepted %s", name) + } + }) + } +} + +// The chart spreads workers with a selector that matches the whole Trino app. +// Across a pool that selector also matches OTHER instances' workers, so the +// instances would compete for nodes and a surge instance might never schedule. +// The term has to be narrowed to this instance, not removed. +func TestInstantiateScopesWorkerAntiAffinityToTheInstance(t *testing.T) { + blueprint := validBlueprint() + blueprint.Worker.PodTemplate.Spec.Affinity = &corev1.Affinity{ + PodAntiAffinity: &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{{ + TopologyKey: "kubernetes.io/hostname", + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app.kubernetes.io/name": "trino"}, + }, + }}, + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{ + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{TopologyKey: "topology.kubernetes.io/zone"}, + }}, + }, + } + + objects, err := blueprint.Instantiate(testIdentity()) + if err != nil { + t.Fatalf("instantiate: %v", err) + } + antiAffinity := objects.WorkerDeployment.Spec.Template.Spec.Affinity.PodAntiAffinity + + required := antiAffinity.RequiredDuringSchedulingIgnoredDuringExecution + if len(required) != 1 { + t.Fatalf("required anti-affinity was dropped: %+v", required) + } + if required[0].LabelSelector.MatchLabels[LabelInstance] != "cell-001-a1b2c3d4" { + t.Fatalf("required term is not instance-scoped: %v", required[0].LabelSelector.MatchLabels) + } + // The chart's own selector and topology key must survive. + if required[0].LabelSelector.MatchLabels["app.kubernetes.io/name"] != "trino" { + t.Fatalf("the chart's selector was replaced: %v", required[0].LabelSelector.MatchLabels) + } + if required[0].TopologyKey != "kubernetes.io/hostname" { + t.Fatalf("topology key = %q", required[0].TopologyKey) + } + + preferred := antiAffinity.PreferredDuringSchedulingIgnoredDuringExecution + if len(preferred) != 1 || preferred[0].Weight != 100 { + t.Fatalf("preferred anti-affinity was dropped or reweighted: %+v", preferred) + } + if preferred[0].PodAffinityTerm.LabelSelector.MatchLabels[LabelInstance] != "cell-001-a1b2c3d4" { + t.Fatal("preferred term is not instance-scoped") + } +} + +// A leader change must not change an instance's spec digest. The epoch is who +// last wrote the object, not what it runs: when it was folded into the digest, +// a leader that created objects and died before the phase CAS left its +// successor computing a different digest, so every later apply was refused as +// a foreign object and the instance - and the whole pool behind it - wedged. +func TestSpecDigestIgnoresTheAuthorityEpoch(t *testing.T) { + blueprint := validBlueprint() + first := testIdentity() + second := testIdentity() + second.AuthorityEpoch = first.AuthorityEpoch + 5 + + if blueprint.SpecDigest(first) != blueprint.SpecDigest(second) { + t.Fatal("a leader change changed the spec digest") + } + // And the objects a successor renders are byte-comparable for ownership. + firstObjects, err := blueprint.Instantiate(first) + if err != nil { + t.Fatalf("instantiate: %v", err) + } + secondObjects, err := blueprint.Instantiate(second) + if err != nil { + t.Fatalf("instantiate: %v", err) + } + if firstObjects.Service.Annotations[AnnotationSpecDigest] != secondObjects.Service.Annotations[AnnotationSpecDigest] { + t.Fatal("the rendered spec-digest annotation changed with the epoch") + } + // The epoch itself still travels, because ownership compares it. + if secondObjects.Service.Annotations[AnnotationAuthorityEpoch] == firstObjects.Service.Annotations[AnnotationAuthorityEpoch] { + t.Fatal("the authority-epoch annotation did not follow the leader") + } +} diff --git a/controlplane/trinopool/phase.go b/controlplane/trinopool/phase.go new file mode 100644 index 000000000..290bb45c5 --- /dev/null +++ b/controlplane/trinopool/phase.go @@ -0,0 +1,124 @@ +package trinopool + +import "fmt" + +// Phase is the durable lifecycle position of one pool instance. It is stored on +// the instance row and is the only thing that authorizes an external effect: +// admission, drain, and above all deletion. +type Phase string + +// The planned lifecycle, then the failure branch. The two are deliberately +// separate: a coordinator that died with in-flight work has NOT drained, and +// reporting it as SEALED/RETIRED would erase that distinction from the record. +const ( + PhasePending Phase = "PENDING" // intent persisted, nothing created yet + PhaseCreating Phase = "CREATING" // Kubernetes objects being created + PhasePreparing Phase = "PREPARING" // pods exist, member registered but unroutable + PhaseValidating Phase = "VALIDATING" // candidate validation in progress + PhaseAdmitted Phase = "ADMITTED" // Gateway CAS accepted; receipt persisted + PhaseServing Phase = "SERVING" // counts toward the minimum serving floor + PhaseDraining Phase = "DRAINING" // no new independent work; obligations remain + PhaseSealed Phase = "SEALED" // obligations finished, retirement not yet claimed + PhaseRetiring Phase = "RETIRING" // irreversible retirement claimed; deletion permitted + PhaseRetired Phase = "RETIRED" // resources verified absent + + PhaseSuspect Phase = "SUSPECT" // probe failing; excluded from new admissions + PhaseLost Phase = "LOST" // authoritative evidence the process terminated + PhaseFailureRetired Phase = "FAILURE_RETIRED" // failure receipt recorded, resources removed + + // PhaseFailedPreparing is a candidate that can never be admitted. It is NOT + // terminal: its Kubernetes objects still exist and its Gateway member is + // still PREPARING, which counts against the pool's live budget. A terminal + // FAILED_PREPARING leaked a whole Trino cluster and, after a single failed + // candidate at desired+surge, no further member could register at all - no + // repair and no rollout. The instance is cleaned up from here and only then + // becomes FAILURE_RETIRED. + PhaseFailedPreparing Phase = "FAILED_PREPARING" +) + +// phaseTransitions is the whole state machine. Absence is denial. +var phaseTransitions = map[Phase][]Phase{ + PhasePending: {PhaseCreating, PhaseFailedPreparing}, + PhaseCreating: {PhasePreparing, PhaseFailedPreparing}, + PhasePreparing: {PhaseValidating, PhaseFailedPreparing}, + PhaseValidating: {PhaseAdmitted, PhasePreparing, PhaseFailedPreparing}, + PhaseAdmitted: {PhaseServing, PhaseDraining, PhaseSuspect}, + PhaseServing: {PhaseDraining, PhaseSuspect}, + PhaseDraining: {PhaseSealed, PhaseSuspect}, + PhaseSealed: {PhaseRetiring, PhaseSuspect}, + PhaseRetiring: {PhaseRetired}, + PhaseRetired: nil, + + // A suspected member always leaves; only the route depends on the + // evidence. There is no path back to SERVING: suspicion is the Gateway's + // state too, and it excludes the member until a fresh certified admission + // - which a recovered-but-uncertain incarnation does not get, because the + // pool can replace it with a certain one instead. Recording a local + // recovery would leave this row claiming a member serves while the Gateway + // refuses to route to it. + PhaseSuspect: {PhaseDraining, PhaseLost}, + PhaseLost: {PhaseFailureRetired}, + PhaseFailureRetired: nil, + // A failed candidate is cleaned up and then recorded as failure-retired. + // The Gateway member it registered is walked PREPARING -> SUSPECT -> LOST + // first, because that is what releases the pool's live slot; deletion is + // permitted before that only because the candidate provably never admitted + // work. + PhaseFailedPreparing: {PhaseFailureRetired}, +} + +// Valid reports whether the phase is one this build knows. A row written by a +// newer binary must not be silently treated as some default. +func (p Phase) Valid() bool { + _, known := phaseTransitions[p] + return known +} + +// Terminal reports whether the phase can never change again. +func (p Phase) Terminal() bool { + return p.Valid() && len(phaseTransitions[p]) == 0 +} + +// Serving reports whether the instance counts toward the minimum serving floor. +// Only SERVING does: an ADMITTED instance has not been observed serving yet and +// a DRAINING one is by definition leaving. +func (p Phase) Serving() bool { return p == PhaseServing } + +// OccupiesCapacity reports whether the instance still holds a live compute slot +// and therefore counts against desired + surge. A terminal tombstone does not: +// historical obligations on a proven-dead process must not block a replacement +// forever. Everything else does, including SUSPECT and LOST, whose pods are +// still running until deletion is verified. +func (p Phase) OccupiesCapacity() bool { return p.Valid() && !p.Terminal() } + +// PermitsResourceDeletion reports whether Kubernetes objects of this instance +// may be deleted. Deletion requires either an irreversible Gateway retirement +// claim (RETIRING and later, FAILURE_RETIRED) or proof the candidate never +// admitted work (FAILED_PREPARING). A transient zero obligation count, a +// failing probe, or a SEALED member without a claim is never sufficient. +func (p Phase) PermitsResourceDeletion() bool { + switch p { + case PhaseRetiring, PhaseRetired, PhaseFailureRetired, PhaseFailedPreparing: + return true + default: + return false + } +} + +// ValidateTransition reports whether the instance may move from one phase to +// another. Callers still have to apply it as a CAS against the stored phase; +// this only decides whether the move is legal at all. +func ValidateTransition(from, to Phase) error { + if !from.Valid() { + return fmt.Errorf("unknown source phase %q", from) + } + if !to.Valid() { + return fmt.Errorf("unknown target phase %q", to) + } + for _, candidate := range phaseTransitions[from] { + if candidate == to { + return nil + } + } + return fmt.Errorf("instance phase %s cannot move to %s", from, to) +} diff --git a/controlplane/trinopool/phase_test.go b/controlplane/trinopool/phase_test.go new file mode 100644 index 000000000..11ceafe84 --- /dev/null +++ b/controlplane/trinopool/phase_test.go @@ -0,0 +1,141 @@ +package trinopool + +import "testing" + +func TestPhaseTransitionsFollowTheLifecycle(t *testing.T) { + allowed := [][2]Phase{ + {PhasePending, PhaseCreating}, + {PhaseCreating, PhasePreparing}, + {PhasePreparing, PhaseValidating}, + {PhaseValidating, PhaseAdmitted}, + {PhaseAdmitted, PhaseServing}, + {PhaseServing, PhaseDraining}, + {PhaseDraining, PhaseSealed}, + {PhaseSealed, PhaseRetiring}, + {PhaseRetiring, PhaseRetired}, + } + for _, step := range allowed { + if err := ValidateTransition(step[0], step[1]); err != nil { + t.Errorf("%s -> %s rejected: %v", step[0], step[1], err) + } + } +} + +// Retirement is the one irreversible claim in the whole design: once Gateway +// has issued it, the instance's incarnation can never serve again. A resume +// would let a retired member take new work. +func TestRetiringNeverResumes(t *testing.T) { + for _, target := range []Phase{PhaseServing, PhaseAdmitted, PhaseDraining, PhaseSealed, PhasePreparing, PhaseValidating} { + if err := ValidateTransition(PhaseRetiring, target); err == nil { + t.Errorf("RETIRING -> %s was allowed", target) + } + if err := ValidateTransition(PhaseRetired, target); err == nil { + t.Errorf("RETIRED -> %s was allowed", target) + } + } +} + +// A drained instance is gone; a lost one is a failure with preserved history. +// Collapsing the two would report a dead coordinator's abandoned queries as a +// successful drain. +func TestFailureBranchIsSeparateFromDrain(t *testing.T) { + if err := ValidateTransition(PhaseServing, PhaseSuspect); err != nil { + t.Fatalf("SERVING -> SUSPECT rejected: %v", err) + } + if err := ValidateTransition(PhaseSuspect, PhaseLost); err != nil { + t.Fatalf("SUSPECT -> LOST rejected: %v", err) + } + if err := ValidateTransition(PhaseLost, PhaseFailureRetired); err != nil { + t.Fatalf("LOST -> FAILURE_RETIRED rejected: %v", err) + } + // A suspected member leaves through the planned drain when it cannot be + // proven dead - a crash-looping coordinator keeps its objects, so a loss + // claim never gets its evidence and the member would otherwise hold its + // slot forever. + if err := ValidateTransition(PhaseSuspect, PhaseDraining); err != nil { + t.Fatalf("SUSPECT -> DRAINING rejected: %v", err) + } + // It does NOT come back locally. The Gateway excluded it and only a fresh + // certified admission un-excludes it, so a local recovery would leave this + // row claiming a member serves while the Gateway routes nothing to it. + if err := ValidateTransition(PhaseSuspect, PhaseServing); err == nil { + t.Fatal("SUSPECT -> SERVING was allowed; a local recovery diverges from the Gateway") + } + // ... but a lost one cannot, and it must never look like a clean drain. + for _, target := range []Phase{PhaseServing, PhaseSealed, PhaseRetired} { + if err := ValidateTransition(PhaseLost, target); err == nil { + t.Errorf("LOST -> %s was allowed", target) + } + } +} + +// A candidate that failed before it was ever admitted is the only instance the +// operator may clean up without a Gateway retirement receipt. +func TestFailedPreparingIsReachableOnlyBeforeAdmission(t *testing.T) { + for _, from := range []Phase{PhasePending, PhaseCreating, PhasePreparing, PhaseValidating} { + if err := ValidateTransition(from, PhaseFailedPreparing); err != nil { + t.Errorf("%s -> FAILED_PREPARING rejected: %v", from, err) + } + } + for _, from := range []Phase{PhaseAdmitted, PhaseServing, PhaseDraining, PhaseSealed} { + if err := ValidateTransition(from, PhaseFailedPreparing); err == nil { + t.Errorf("%s -> FAILED_PREPARING was allowed after admission", from) + } + } +} + +func TestPhaseClassification(t *testing.T) { + // Serving capacity is what the minimum-serving floor counts. + if !PhaseServing.Serving() || PhaseDraining.Serving() || PhaseAdmitted.Serving() { + t.Error("serving classification is wrong") + } + // Live compute is what the surge budget counts: anything that occupies a + // pod, including a draining or suspect instance. + // + // FAILED_PREPARING is in this group deliberately: the candidate's pods are + // still running and its Gateway member is still PREPARING, which the + // Gateway counts as live. Treating it as a tombstone hid a whole leaked + // cluster and let one failed candidate exhaust the registration budget. + for _, phase := range []Phase{PhasePending, PhaseCreating, PhasePreparing, PhaseValidating, PhaseAdmitted, PhaseServing, PhaseDraining, PhaseSealed, PhaseRetiring, PhaseSuspect, PhaseLost, PhaseFailedPreparing} { + if !phase.OccupiesCapacity() { + t.Errorf("%s should occupy capacity", phase) + } + } + // A failed candidate is cleaned up rather than abandoned. + if PhaseFailedPreparing.Terminal() { + t.Error("FAILED_PREPARING must be able to reach FAILURE_RETIRED, or its resources and Gateway member leak") + } + // A historical tombstone must not consume a live slot forever. + for _, phase := range []Phase{PhaseRetired, PhaseFailureRetired} { + if phase.OccupiesCapacity() { + t.Errorf("%s should not occupy capacity", phase) + } + if !phase.Terminal() { + t.Errorf("%s should be terminal", phase) + } + } +} + +// Deleting Kubernetes objects before Gateway has irreversibly claimed the +// incarnation can destroy running queries. +func TestOnlyRetirementPhasesPermitDeletion(t *testing.T) { + for _, phase := range []Phase{PhaseRetiring, PhaseRetired, PhaseFailureRetired, PhaseFailedPreparing} { + if !phase.PermitsResourceDeletion() { + t.Errorf("%s should permit deletion", phase) + } + } + for _, phase := range []Phase{PhasePending, PhaseCreating, PhasePreparing, PhaseValidating, PhaseAdmitted, PhaseServing, PhaseDraining, PhaseSealed, PhaseSuspect, PhaseLost} { + if phase.PermitsResourceDeletion() { + t.Errorf("%s must not permit deletion", phase) + } + } +} + +func TestUnknownPhaseIsRejected(t *testing.T) { + if err := ValidateTransition(Phase("BANANA"), PhaseServing); err == nil { + t.Fatal("an unknown phase was accepted") + } + if Phase("BANANA").Valid() { + t.Fatal("an unknown phase reported itself valid") + } +} diff --git a/controlplane/trinopool/plan.go b/controlplane/trinopool/plan.go new file mode 100644 index 000000000..4abccc049 --- /dev/null +++ b/controlplane/trinopool/plan.go @@ -0,0 +1,211 @@ +package trinopool + +import "sort" + +// PlanAction is the single lifecycle step the operator may take for a pool on +// one tick. One step at a time is deliberate: a stuck rollout must not be able +// to spawn a chain of replacements while nobody is looking. +type PlanAction string + +const ( + PlanActionNone PlanAction = "none" + PlanActionCreate PlanAction = "create" + PlanActionDrain PlanAction = "drain" +) + +// InstanceView is the planner's read-only projection of an instance row. +type InstanceView struct { + ID string + Phase Phase + ReleaseID string + // Repair marks an instance created against the failure-repair budget rather + // than the planned surge budget. The two budgets are separate so a failure + // during a release rollout does not stall capacity restoration. + Repair bool + // CreatedAt orders drain candidates deterministically; zero is fine, ID is + // the tiebreaker. + CreatedAt int64 +} + +// PoolState is everything the planner needs. It carries no clients and no +// clock, so every decision is reproducible in a test. +type PoolState struct { + DesiredInstances int + MinServing int + MaxSurge int + MaxRepair int + DesiredReleaseID string + // Frozen is set when desired configuration is missing or invalid. The pool + // then holds its last-good state: no creates, no drains, no deletes. + Frozen bool + FrozenReason string + Instances []InstanceView +} + +// Plan is the decided step plus the reason, which is surfaced to operators. +// A blocked plan always explains itself: "nothing happened" is the hardest +// state to debug from metrics alone. +type Plan struct { + Action PlanAction + InstanceID string + Repair bool + // RepairFor names the failed instance a repair replaces. The Gateway + // charges an activation to the repair budget only when it is set, so a + // repair without it silently spends the single planned surge instead. + RepairFor string + Reason string +} + +// PlanNext decides the one step to take. The order of the rules is the +// contract: +// +// 1. A frozen or unconfigured pool does nothing at all. +// 2. Restoring capacity below the desired count outranks upgrading a release. +// 3. Only one create may be in flight, and only one drain. +// 4. A drain happens only once the replacement is actually serving and the +// serving floor survives the departure. +// 5. Nothing is ever forced to free a budget. +func PlanNext(state PoolState) Plan { + if state.Frozen { + return Plan{Action: PlanActionNone, Reason: blockedReason("pool is frozen", state.FrozenReason)} + } + // A zero desired count is missing configuration, never an instruction to + // empty the pool. + if state.DesiredInstances < 1 { + return Plan{Action: PlanActionNone, Reason: "pool has no desired instance count"} + } + + var live, healthy, serving, preparing, repairing, draining int + for _, instance := range state.Instances { + if !instance.Phase.OccupiesCapacity() { + continue + } + live++ + switch instance.Phase { + case PhaseSuspect, PhaseLost, PhaseFailedPreparing: + // Still holds a pod, but cannot be counted on to serve. A failed + // candidate is in this group until its cleanup completes: counting + // it as healthy would hide the capacity deficit it caused. + default: + healthy++ + } + if instance.Phase.Serving() { + serving++ + } + if preServing(instance.Phase) { + preparing++ + if instance.Repair { + repairing++ + } + } + if instance.Phase == PhaseDraining || instance.Phase == PhaseSealed || instance.Phase == PhaseRetiring { + draining++ + } + } + + // 2. Capacity deficit. Counting `preparing` here is what keeps a slow + // replacement from being duplicated every tick. + if deficit := state.DesiredInstances - healthy; deficit > preparing { + // Filling an empty pool needs no special budget. Only a deficit caused + // by instances that still HOLD a slot while being unable to serve + // (SUSPECT/LOST, pods not yet verified absent) needs the repair budget, + // because the replacement necessarily runs above the desired count. + if live < state.DesiredInstances { + return Plan{Action: PlanActionCreate, Reason: "creating an instance to reach the desired instance count"} + } + if repairing >= state.MaxRepair { + return Plan{Action: PlanActionNone, Reason: "capacity is short but the repair budget is exhausted; investigate the failed instances"} + } + if live >= state.DesiredInstances+state.MaxSurge+state.MaxRepair { + return Plan{Action: PlanActionNone, Reason: "capacity is short but no live compute slot is free; investigate the failed instances"} + } + return Plan{ + Action: PlanActionCreate, Repair: true, + RepairFor: repairTarget(state), + Reason: "restoring capacity lost to a failed instance", + } + } + + // 3. One lifecycle operation at a time. + if preparing > 0 { + return Plan{Action: PlanActionNone, Reason: "an instance is already preparing"} + } + if draining > 0 { + return Plan{Action: PlanActionNone, Reason: "an instance is already draining"} + } + + outdated := outdatedServing(state) + if len(outdated) == 0 { + return Plan{Action: PlanActionNone, Reason: "pool matches the desired release and instance count"} + } + + // 4. Drain only when the floor survives it. + if serving-1 >= state.MinServing { + return Plan{Action: PlanActionDrain, InstanceID: outdated[0].ID, Reason: "replacing an instance running an older release"} + } + + // 5. Otherwise surge one replacement, within the surge budget. + if live >= state.DesiredInstances+state.MaxSurge { + return Plan{Action: PlanActionNone, Reason: "release rollout is waiting for the surge budget to free up"} + } + return Plan{Action: PlanActionCreate, Reason: "surging a replacement for an instance running an older release"} +} + +// outdatedServing lists serving instances that do not run the desired release, +// oldest first so replacement order is deterministic across leaders. +func outdatedServing(state PoolState) []InstanceView { + var outdated []InstanceView + for _, instance := range state.Instances { + if instance.Phase.Serving() && instance.ReleaseID != state.DesiredReleaseID { + outdated = append(outdated, instance) + } + } + sort.Slice(outdated, func(i, j int) bool { + if outdated[i].CreatedAt != outdated[j].CreatedAt { + return outdated[i].CreatedAt < outdated[j].CreatedAt + } + return outdated[i].ID < outdated[j].ID + }) + return outdated +} + +func preServing(phase Phase) bool { + switch phase { + case PhasePending, PhaseCreating, PhasePreparing, PhaseValidating, PhaseAdmitted: + return true + default: + return false + } +} + +func blockedReason(base, detail string) string { + if detail == "" { + return base + } + return base + ": " + detail +} + +// repairTarget picks the instance a repair replaces: a PROVEN failure first, +// because a SUSPECT member may still recover and charging the repair budget for +// it would spend a budget on a member that never failed. Oldest first, so the +// choice is stable across leaders. +func repairTarget(state PoolState) string { + var suspect string + var lost string + for _, instance := range state.Instances { + switch instance.Phase { + case PhaseLost: + if lost == "" || instance.ID < lost { + lost = instance.ID + } + case PhaseSuspect: + if suspect == "" || instance.ID < suspect { + suspect = instance.ID + } + } + } + if lost != "" { + return lost + } + return suspect +} diff --git a/controlplane/trinopool/plan_test.go b/controlplane/trinopool/plan_test.go new file mode 100644 index 000000000..5e13e3a08 --- /dev/null +++ b/controlplane/trinopool/plan_test.go @@ -0,0 +1,193 @@ +package trinopool + +import "testing" + +func servingPool(release string, count int) PoolState { + state := PoolState{DesiredInstances: 3, MinServing: 3, MaxSurge: 1, MaxRepair: 1, DesiredReleaseID: release} + for index := 0; index < count; index++ { + state.Instances = append(state.Instances, InstanceView{ + ID: string(rune('a'+index)) + "-instance", Phase: PhaseServing, ReleaseID: release, + }) + } + return state +} + +func TestPlanCreatesUpToDesiredCount(t *testing.T) { + plan := PlanNext(servingPool("r1", 1)) + if plan.Action != PlanActionCreate || plan.Repair { + t.Fatalf("expected a plain create, got %+v", plan) + } +} + +func TestPlanIsSatisfiedAtDesiredCount(t *testing.T) { + if plan := PlanNext(servingPool("r1", 3)); plan.Action != PlanActionNone { + t.Fatalf("expected no action at the desired count, got %+v", plan) + } +} + +// Missing or unreadable desired configuration must freeze the pool at its +// last-good state. It must never be read as "desired count zero", which would +// delete the fleet. +func TestFrozenPoolTakesNoAction(t *testing.T) { + state := servingPool("r1", 1) + state.Frozen = true + if plan := PlanNext(state); plan.Action != PlanActionNone { + t.Fatalf("a frozen pool planned %+v", plan) + } +} + +func TestPlanNeverActsOnAnUnconfiguredPool(t *testing.T) { + // A zero-valued desired count is missing configuration, not an instruction. + state := servingPool("r1", 3) + state.DesiredInstances = 0 + if plan := PlanNext(state); plan.Action != PlanActionNone { + t.Fatalf("an unconfigured pool planned %+v", plan) + } +} + +// One lifecycle operation at a time: a second create while one is preparing +// would let a stuck rollout spawn an unbounded chain of instances. +func TestPlanWaitsForAnInFlightCreate(t *testing.T) { + state := servingPool("r1", 1) + state.Instances = append(state.Instances, InstanceView{ID: "new", Phase: PhasePreparing, ReleaseID: "r1"}) + if plan := PlanNext(state); plan.Action != PlanActionNone { + t.Fatalf("expected the planner to wait, got %+v", plan) + } +} + +func TestPlanWaitsForAnInFlightDrain(t *testing.T) { + state := servingPool("r2", 3) + state.Instances[0].ReleaseID = "r1" + state.Instances = append(state.Instances, InstanceView{ID: "new", Phase: PhaseDraining, ReleaseID: "r2"}) + if plan := PlanNext(state); plan.Action != PlanActionNone { + t.Fatalf("expected the planner to wait for the drain, got %+v", plan) + } +} + +// A new release surges one instance above the desired count, and only one. +func TestPlanSurgesOneInstanceForANewRelease(t *testing.T) { + state := servingPool("r1", 3) + state.DesiredReleaseID = "r2" + plan := PlanNext(state) + if plan.Action != PlanActionCreate || plan.Repair { + t.Fatalf("expected a surge create, got %+v", plan) + } +} + +func TestPlanDrainsTheOutdatedInstanceOnlyOnceTheSurgeIsServing(t *testing.T) { + state := servingPool("r1", 3) + state.DesiredReleaseID = "r2" + state.Instances = append(state.Instances, InstanceView{ID: "surge", Phase: PhaseServing, ReleaseID: "r2"}) + + plan := PlanNext(state) + if plan.Action != PlanActionDrain { + t.Fatalf("expected a drain, got %+v", plan) + } + if plan.InstanceID == "surge" { + t.Fatal("the planner drained the new instance instead of an outdated one") + } +} + +// The floor is the whole point of the surge: a planned drain may never take the +// ready serving count below the minimum. +func TestPlanNeverDrainsBelowTheServingFloor(t *testing.T) { + state := servingPool("r1", 3) + state.DesiredReleaseID = "r2" + // No surge instance exists, so draining now would leave two serving. + for _, instance := range state.Instances { + if instance.Phase == PhaseServing && instance.ReleaseID != "r2" { + // sanity: the fixture really is outdated + goto check + } + } + t.Fatal("fixture is not outdated") +check: + if plan := PlanNext(state); plan.Action == PlanActionDrain { + t.Fatalf("planner drained below the serving floor: %+v", plan) + } +} + +// Restoring lost capacity outranks upgrading a release. +func TestPlanRepairsLostCapacityBeforeUpgrading(t *testing.T) { + state := servingPool("r1", 3) + state.DesiredReleaseID = "r2" + state.Instances[0].Phase = PhaseLost + plan := PlanNext(state) + if plan.Action != PlanActionCreate || !plan.Repair { + t.Fatalf("expected a repair create, got %+v", plan) + } +} + +// The repair budget is one extra live instance, not an open-ended supply: a +// flapping cluster must block and alert rather than spawn forever. +func TestPlanBoundsTheRepairBudget(t *testing.T) { + state := servingPool("r1", 3) + for index := range state.Instances { + state.Instances[index].Phase = PhaseLost + } + state.Instances = append(state.Instances, InstanceView{ID: "repair", Phase: PhasePreparing, ReleaseID: "r1", Repair: true}) + plan := PlanNext(state) + if plan.Action != PlanActionNone { + t.Fatalf("expected the repair budget to bound the plan, got %+v", plan) + } + if plan.Reason == "" { + t.Fatal("a blocked plan must carry a reason for the operator") + } +} + +// A terminal tombstone records history; it must not hold a live compute slot. +func TestRetiredInstancesDoNotConsumeCapacity(t *testing.T) { + state := servingPool("r1", 3) + for index := 0; index < 5; index++ { + state.Instances = append(state.Instances, InstanceView{ID: "old", Phase: PhaseRetired, ReleaseID: "r0"}) + } + if plan := PlanNext(state); plan.Action != PlanActionNone { + t.Fatalf("tombstones changed the plan: %+v", plan) + } +} + +func TestPlanReportsBlockedSurge(t *testing.T) { + // Desired release changed, the surge slot is already spent by a stuck + // instance that is serving but still outdated: nothing may be forced. + state := PoolState{DesiredInstances: 3, MinServing: 3, MaxSurge: 1, MaxRepair: 1, DesiredReleaseID: "r2"} + for index := 0; index < 4; index++ { + state.Instances = append(state.Instances, InstanceView{ID: string(rune('a' + index)), Phase: PhaseServing, ReleaseID: "r1"}) + } + plan := PlanNext(state) + if plan.Action != PlanActionDrain { + t.Fatalf("expected the extra outdated instance to be drained, got %+v", plan) + } +} + +// A repair has to NAME the instance it replaces. The Gateway charges an +// activation to the repair budget only when repairFor points at a failed +// member; without it the repair spends the single planned surge, so a failure +// during a release rollout cannot be repaired at all. +func TestRepairPlanNamesTheInstanceItReplaces(t *testing.T) { + state := servingPool("r1", 3) + state.Instances[1].ID = "broken-one" + state.Instances[1].Phase = PhaseLost + + plan := PlanNext(state) + if plan.Action != PlanActionCreate || !plan.Repair { + t.Fatalf("expected a repair create, got %+v", plan) + } + if plan.RepairFor != "broken-one" { + t.Fatalf("repair names %q, want the failed instance", plan.RepairFor) + } +} + +// A SUSPECT instance is not yet proven dead, so it is not yet a repair target: +// naming it would charge the repair budget for a member that may still recover. +func TestRepairPrefersAProvenFailure(t *testing.T) { + state := servingPool("r1", 3) + state.Instances[0].ID = "suspected" + state.Instances[0].Phase = PhaseSuspect + state.Instances[1].ID = "lost-one" + state.Instances[1].Phase = PhaseLost + + plan := PlanNext(state) + if plan.RepairFor != "lost-one" { + t.Fatalf("repair names %q, want the lost instance", plan.RepairFor) + } +} diff --git a/controlplane/trinopool/testdata/blueprint.json b/controlplane/trinopool/testdata/blueprint.json new file mode 100644 index 000000000..0a8a99d1a --- /dev/null +++ b/controlplane/trinopool/testdata/blueprint.json @@ -0,0 +1,102 @@ +{ + "blueprint_version": 1, + "release_id": "2026-09-18.1", + "chart_version": "trino-0.1.0", + "image": "registry.example.invalid/trino@sha256:1111111111111111111111111111111111111111111111111111111111111111", + "namespace": "trino-pool-example", + "service_account_name": "trino", + "coordinator": { + "pod_template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "trino", + "app.kubernetes.io/instance": "trino-pool-example" + } + }, + "spec": { + "serviceAccountName": "trino", + "terminationGracePeriodSeconds": 3600, + "securityContext": {"runAsNonRoot": true, "runAsUser": 1000}, + "containers": [ + { + "name": "trino-coordinator", + "image": "registry.example.invalid/trino@sha256:1111111111111111111111111111111111111111111111111111111111111111", + "ports": [{"name": "https", "containerPort": 8443, "protocol": "TCP"}], + "env": [ + { + "name": "TRINO_INTERNAL_COMMUNICATION_SHARED_SECRET", + "valueFrom": {"secretKeyRef": {"name": "trino-internal-communication", "key": "shared-secret"}} + } + ], + "volumeMounts": [ + {"name": "config", "mountPath": "/etc/trino/config.properties", "subPath": "config.properties"}, + {"name": "config", "mountPath": "/etc/trino/node.properties", "subPath": "node.properties"}, + {"name": "trino-auth", "mountPath": "/etc/trino/auth", "readOnly": true}, + {"name": "data", "mountPath": "/data/trino"} + ], + "resources": {"requests": {"cpu": "2", "memory": "16Gi"}, "limits": {"memory": "16Gi"}} + }, + { + "name": "opa", + "image": "registry.example.invalid/opa@sha256:2222222222222222222222222222222222222222222222222222222222222222", + "args": ["run", "--server", "--addr=0.0.0.0:8181", "--config-file=/etc/opa/config.yaml"], + "ports": [{"name": "opa-http", "containerPort": 8181, "protocol": "TCP"}] + } + ], + "volumes": [ + {"name": "trino-auth", "secret": {"secretName": "trino-auth", "optional": true}}, + {"name": "data", "emptyDir": {}} + ] + } + } + }, + "worker": { + "replicas": 4, + "pod_template": { + "metadata": {"labels": {"app.kubernetes.io/name": "trino"}}, + "spec": { + "serviceAccountName": "trino", + "containers": [ + { + "name": "trino-worker", + "image": "registry.example.invalid/trino@sha256:1111111111111111111111111111111111111111111111111111111111111111", + "ports": [{"name": "https", "containerPort": 8443, "protocol": "TCP"}], + "volumeMounts": [ + {"name": "config", "mountPath": "/etc/trino/config.properties", "subPath": "config.properties"}, + {"name": "config", "mountPath": "/etc/trino/node.properties", "subPath": "node.properties"}, + {"name": "data", "mountPath": "/data/trino"} + ], + "resources": {"requests": {"cpu": "4", "memory": "32Gi"}, "limits": {"memory": "32Gi"}} + } + ], + "volumes": [{"name": "data", "emptyDir": {}}] + } + } + }, + "config_files": { + "coordinator": { + "config.properties": "coordinator=true\nnode-scheduler.include-coordinator=false\nhttp-server.http.port=8080\ndiscovery.uri=${ENV:TRINO_DISCOVERY_URI}\n", + "node.properties": "node.environment=${ENV:TRINO_NODE_ENVIRONMENT}\nnode.data-dir=/data/trino\n", + "jvm.config": "-server\n-XX:+UseG1GC\n" + }, + "worker": { + "config.properties": "coordinator=false\nhttp-server.http.port=8080\ndiscovery.uri=${ENV:TRINO_DISCOVERY_URI}\n", + "node.properties": "node.environment=${ENV:TRINO_NODE_ENVIRONMENT}\nnode.data-dir=/data/trino\n", + "jvm.config": "-server\n-XX:+UseG1GC\n" + } + }, + "shared_resources": { + "secrets": ["trino-auth", "trino-tenant-secrets", "trino-internal-communication", "trino-opa-bundle-token"], + "config_maps": ["trino-resource-groups"] + }, + "identity_binding": { + "instance_label": "posthog.com/trino-instance", + "component_label": "app.kubernetes.io/component", + "discovery_uri_env": "TRINO_DISCOVERY_URI", + "node_environment_env": "TRINO_NODE_ENVIRONMENT", + "instance_id_env": "DUCKGRES_TRINO_INSTANCE_ID", + "coordinator_http_port_name": "https", + "coordinator_container_name": "trino-coordinator", + "worker_container_name": "trino-worker" + } +} diff --git a/tests/configstore/helpers_test.go b/tests/configstore/helpers_test.go index e8a42902a..591e16516 100644 --- a/tests/configstore/helpers_test.go +++ b/tests/configstore/helpers_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -39,13 +40,26 @@ func newIsolatedConfigStore(t *testing.T) *cpconfigstore.ConfigStore { return store } -func newIsolatedConfigStoreSchema(t *testing.T) (*sql.DB, string) { +// baseConfigStoreDSN is the container-backed default. DUCKGRES_TEST_PG_DSN +// points the suite at an already-running PostgreSQL instead, which is how these +// tests run where Docker is unavailable. +func baseConfigStoreDSN(t *testing.T) string { t.Helper() + if dsn := strings.TrimSpace(os.Getenv("DUCKGRES_TEST_PG_DSN")); dsn != "" { + return dsn + } ensureIntegrationPostgres(t) + return "host=127.0.0.1 port=35432 user=postgres password=postgres dbname=testdb sslmode=disable" +} + +func newIsolatedConfigStoreSchema(t *testing.T) (*sql.DB, string) { + t.Helper() + + dsn := baseConfigStoreDSN(t) schema := fmt.Sprintf("managed_warehouse_%d", time.Now().UnixNano()) - adminDB, err := sql.Open("postgres", "host=127.0.0.1 port=35432 user=postgres password=postgres dbname=testdb sslmode=disable") + adminDB, err := sql.Open("postgres", dsn) if err != nil { t.Fatalf("open postgres admin db: %v", err) } @@ -60,8 +74,7 @@ func newIsolatedConfigStoreSchema(t *testing.T) (*sql.DB, string) { _, _ = adminDB.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) }) - connStr := "host=127.0.0.1 port=35432 user=postgres password=postgres dbname=testdb sslmode=disable search_path=" + schema - return adminDB, connStr + return adminDB, dsn + " search_path=" + schema } func cpconfigStoreNew(connStr string) (*cpconfigstore.ConfigStore, error) { diff --git a/tests/configstore/migrations_postgres_test.go b/tests/configstore/migrations_postgres_test.go index 8f22160f5..7aa29ca57 100644 --- a/tests/configstore/migrations_postgres_test.go +++ b/tests/configstore/migrations_postgres_test.go @@ -58,13 +58,68 @@ func TestConfigStoreRunsVersionedSQLMigrations(t *testing.T) { requireGooseMigrationRecorded(t, db, 38) requireGooseMigrationRecorded(t, db, 39) requireGooseMigrationRecorded(t, db, 40) - requireGooseLatestVersion(t, db, 40) + requireGooseMigrationRecorded(t, db, 41) + requireGooseLatestVersion(t, db, 41) requireTablePresent(t, db, "duckgres_trino_cell_lifecycle") for _, column := range []string{"reconcile_owner", "reconcile_epoch", "intent_sequence", "intent", "admission_epoch", "freeze_operation_id", "freeze_stable", "certificate"} { requireColumnPresent(t, db, "duckgres_trino_cell_lifecycle", column) } requireTableAbsent(t, db, "duckgres_schema_migrations") + // Migration 000041 added the shared Trino compute pool tables. They stay + // empty and unread while the feature flags are off. + requireTablePresent(t, db, "duckgres_trino_pools") + // The desired generation orders configuration: the authority epoch says + // who may write, not whether what they hold is current. + requireColumnPresent(t, db, "duckgres_trino_pools", "desired_generation") + for _, column := range []string{"api_mode", "desired_release_id", "desired_instances", "min_serving", "max_surge", "max_repair", "authority_epoch", "authority_owner", "publication_revision", "admitted_revision", "frozen", "frozen_reason"} { + requireColumnPresent(t, db, "duckgres_trino_pools", column) + } + requireTablePresent(t, db, "duckgres_trino_pool_instances") + // Instance inventory includes worker config maps, the repair target, and the failure reason. + for _, column := range []string{"release_id", "spec_digest", "blueprint_snapshot", "phase", "owner_epoch", "repair", "repair_for", "failure_reason", "coordinator_deployment_uid", "worker_deployment_uid", "service_uid", "config_map_uid", "worker_config_map_name", "worker_config_map_uid", "coordinator_pod_uid", "coordinator_node_id", "coordinator_boot_id", "endpoint_url", "gateway_incarnation", "gateway_generation", "applied_catalog_revision", "validation_receipt", "retirement_receipt", + // The coordinator identity the GATEWAY observed at + // registration. A loss claim must present it exactly, or a failed member + // keeps its live slot forever. + "coordinator_id", + // The container instance the admitted process ran in. + // A termination record names a container instance, so without this + // there is nothing to correlate one with, and a restart from before + // admission reads exactly like the death of the admitted process. + "coordinator_container_id"} { + requireColumnPresent(t, db, "duckgres_trino_pool_instances", column) + } + requireTablePresent(t, db, "duckgres_trino_pool_operations") + for _, column := range []string{"intent_hash", "owner_epoch", "attempts", "next_attempt_at", "terminal_at"} { + requireColumnPresent(t, db, "duckgres_trino_pool_operations", column) + } + requireTablePresent(t, db, "duckgres_trino_pool_operation_steps") + requireColumnPresent(t, db, "duckgres_trino_pool_operation_steps", "payload_hash") + requireTablePresent(t, db, "duckgres_trino_pool_publications") + // The barrier's own revision strings record which binding + // was published, which target is open, and which one committed. They are + // durable because an in-memory record cannot survive a leadership move. + for _, column := range []string{"desired_revision", "published_revision", "admitted_revision", "publication_id", "state", "gateway_receipt", + "principal_revision", "target_revision", "admitted_target_revision", + // Which attempt is in flight for this tenant, and the + // durable per-tenant backoff that keeps one failing warehouse from + // starving the queue the driver walks one tenant at a time. + "attempt", "attempts", "next_attempt_at", + // Which request the open occurrence stands for while + // its outcome is unknown, so a request still executing at the Gateway + // cannot commit after a newer intent has been checkpointed here. + "pending_intent", + // And the request itself, so the reissue is byte-identical rather than + // a new body under an old identity. Principal identifiers only; no + // credential material ever enters it. + "pending_payload"} { + requireColumnPresent(t, db, "duckgres_trino_pool_publications", column) + } + requireTablePresent(t, db, "duckgres_trino_pool_projection") + for _, column := range []string{"authority_epoch", "accepted_revision", "accepted_digest"} { + requireColumnPresent(t, db, "duckgres_trino_pool_projection", column) + } + // Migration 000018 added the reshard operation + verbose log tables. requireTablePresent(t, db, "duckgres_reshard_operations") requireColumnPresent(t, db, "duckgres_reshard_operations", "source_kind") @@ -326,8 +381,14 @@ func TestConfigStoreSQLMigrationsUpgradeVersion8Schema(t *testing.T) { DROP TABLE IF EXISTS duckgres_managed_warehouse_trino; DROP TABLE IF EXISTS duckgres_trino_cluster_bootstrap; DROP TABLE IF EXISTS duckgres_trino_cell_lifecycle; -DROP FUNCTION IF EXISTS duckgres_select_trino_backend_on_enable(); - DELETE FROM goose_db_version WHERE version_id IN (9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40); + DROP TABLE IF EXISTS duckgres_trino_pool_projection; + DROP TABLE IF EXISTS duckgres_trino_pool_publications; + DROP TABLE IF EXISTS duckgres_trino_pool_operation_steps; + DROP TABLE IF EXISTS duckgres_trino_pool_operations; + DROP TABLE IF EXISTS duckgres_trino_pool_instances; + DROP TABLE IF EXISTS duckgres_trino_pools; + DROP FUNCTION IF EXISTS duckgres_select_trino_backend_on_enable(); + DELETE FROM goose_db_version WHERE version_id IN (9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41); `).Error; err != nil { t.Fatalf("downgrade baseline schema to pre-v9 shape: %v", err) } @@ -378,7 +439,8 @@ DROP FUNCTION IF EXISTS duckgres_select_trino_backend_on_enable(); requireGooseMigrationRecorded(t, upgradedDB, 35) requireGooseMigrationRecorded(t, upgradedDB, 36) requireGooseMigrationRecorded(t, upgradedDB, 38) - requireGooseLatestVersion(t, upgradedDB, 40) + requireGooseMigrationRecorded(t, upgradedDB, 41) + requireGooseLatestVersion(t, upgradedDB, 41) requireColumnPresent(t, upgradedDB, "duckgres_reshard_operations", "password_url") requireTablePresent(t, upgradedDB, "duckgres_worker_spawn_log") requireColumnDefault(t, upgradedDB, "duckgres_orgs", "max_vcpus", "0") @@ -428,8 +490,14 @@ func TestConfigStoreSQLMigration34VersionsExistingAndNewOrgs(t *testing.T) { DROP TABLE IF EXISTS duckgres_managed_warehouse_trino; DROP TABLE IF EXISTS duckgres_trino_cluster_bootstrap; DROP TABLE IF EXISTS duckgres_trino_cell_lifecycle; -DROP FUNCTION IF EXISTS duckgres_select_trino_backend_on_enable(); - DELETE FROM goose_db_version WHERE version_id IN (34, 35, 36, 37, 38, 39, 40); + DROP TABLE IF EXISTS duckgres_trino_pool_projection; + DROP TABLE IF EXISTS duckgres_trino_pool_publications; + DROP TABLE IF EXISTS duckgres_trino_pool_operation_steps; + DROP TABLE IF EXISTS duckgres_trino_pool_operations; + DROP TABLE IF EXISTS duckgres_trino_pool_instances; + DROP TABLE IF EXISTS duckgres_trino_pools; + DROP FUNCTION IF EXISTS duckgres_select_trino_backend_on_enable(); + DELETE FROM goose_db_version WHERE version_id IN (34, 35, 36, 37, 38, 39, 40, 41); `).Error; err != nil { t.Fatalf("restore pre-migration-34 schema: %v", err) } @@ -1350,7 +1418,15 @@ func TestConfigStoreMigration40PinsExistingTrinoBackends(t *testing.T) { DROP TRIGGER duckgres_select_trino_backend_on_enable ON duckgres_managed_warehouse_trino; DROP FUNCTION duckgres_select_trino_backend_on_enable(); ALTER TABLE duckgres_managed_warehouse_trino DROP COLUMN backend, DROP COLUMN backend_selected, DROP COLUMN hoglake_initialized; - DELETE FROM goose_db_version WHERE version_id=40; + -- Rewind the shared pool migration with 40 so Goose can reapply both in order. + -- The pool tables are self-contained; dropping them restores the pre-40 schema. + DROP TABLE IF EXISTS duckgres_trino_pool_projection; + DROP TABLE IF EXISTS duckgres_trino_pool_publications; + DROP TABLE IF EXISTS duckgres_trino_pool_operation_steps; + DROP TABLE IF EXISTS duckgres_trino_pool_operations; + DROP TABLE IF EXISTS duckgres_trino_pool_instances; + DROP TABLE IF EXISTS duckgres_trino_pools; + DELETE FROM goose_db_version WHERE version_id >= 40; INSERT INTO duckgres_orgs (name,database_name) VALUES ('old-enabled','old_enabled'),('old-disabled','old_disabled'),('old-cell-only','old_cell_only'); INSERT INTO duckgres_managed_warehouse_trino (org_id,enabled) VALUES ('old-enabled',TRUE),('old-disabled',FALSE),('old-cell-only',FALSE); `).Error; err != nil { diff --git a/tests/configstore/trino_pool_postgres_test.go b/tests/configstore/trino_pool_postgres_test.go new file mode 100644 index 000000000..dbf636c47 --- /dev/null +++ b/tests/configstore/trino_pool_postgres_test.go @@ -0,0 +1,980 @@ +//go:build linux || darwin + +package configstore_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "sync" + "testing" + "time" + + cpconfigstore "github.com/posthog/duckgres/controlplane/configstore" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +const poolID = "registered:cell-001" + +func poolSpec() cpconfigstore.TrinoPoolSpec { + return cpconfigstore.TrinoPoolSpec{ + PoolID: poolID, + PublicID: "cell-001", + APIMode: cpconfigstore.TrinoPoolAPIModeShared, + DesiredReleaseID: "r1", + DesiredBlueprintDigest: "d1", + DesiredInstances: 3, + MinServing: 3, + MaxSurge: 1, + MaxRepair: 1, + } +} + +func newPoolStore(t *testing.T) *cpconfigstore.ConfigStore { + t.Helper() + store := newIsolatedConfigStore(t) + // Seeding is the one unfenced write: a fence needs a row to lock, so the + // first publication has to create one. It can only INSERT. + if err := store.SeedTrinoPool(context.Background(), poolSpec()); err != nil { + t.Fatalf("seed pool: %v", err) + } + return store +} + +func claimPool(t *testing.T, store *cpconfigstore.ConfigStore, owner string) cpconfigstore.TrinoPoolLease { + t.Helper() + lease, err := store.AcquireTrinoPoolAuthority(context.Background(), poolID, owner) + if err != nil { + t.Fatalf("acquire authority: %v", err) + } + return lease +} + +func newInstance(id string, phase trinopool.Phase) cpconfigstore.TrinoPoolInstanceSpec { + return cpconfigstore.TrinoPoolInstanceSpec{ + InstanceID: id, + PoolID: poolID, + ReleaseID: "r1", + SpecDigest: "spec-" + id, + BlueprintSnapshot: `{"blueprint_version":1}`, + Phase: phase, + } +} + +// The desired spec is upserted from configuration on every startup; it must not +// disturb the authority epoch, the freeze flag or anything the operator owns. +func TestUpsertTrinoPoolSpecPreservesRuntimeState(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + if err := store.FreezeTrinoPool(ctx, lease, poolID, "blueprint unreadable"); err != nil { + t.Fatalf("freeze: %v", err) + } + spec := poolSpec() + spec.DesiredReleaseID = "r2" + if err := store.UpsertTrinoPoolSpec(ctx, lease, spec); err != nil { + t.Fatalf("upsert: %v", err) + } + + pool, err := store.GetTrinoPool(ctx, poolID) + if err != nil || pool == nil { + t.Fatalf("get pool: %v", err) + } + if pool.DesiredReleaseID != "r2" { + t.Fatalf("desired release = %q, want r2", pool.DesiredReleaseID) + } + if pool.AuthorityEpoch != lease.Epoch { + t.Fatalf("epoch changed to %d, want %d", pool.AuthorityEpoch, lease.Epoch) + } + if !pool.Frozen || pool.FrozenReason == "" { + t.Fatal("a config upsert cleared the freeze") + } +} + +// Publishing desired state is lifecycle-affecting, so it is fenced: a delayed +// old leader must not be able to overwrite a newer desired spec or clear +// another leader's freeze. +func TestDesiredStatePublicationIsFenced(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + stale := claimPool(t, store, "cp-a") + current := claimPool(t, store, "cp-b") + + spec := poolSpec() + spec.DesiredReleaseID = "stale-release" + if err := store.UpsertTrinoPoolSpec(ctx, stale, spec); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale desired publication error = %v, want ErrTrinoPoolConflict", err) + } + if err := store.FreezeTrinoPool(ctx, stale, poolID, "stale freeze"); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale freeze error = %v, want ErrTrinoPoolConflict", err) + } + if err := store.ThawTrinoPool(ctx, stale, poolID); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale thaw error = %v, want ErrTrinoPoolConflict", err) + } + + pool, err := store.GetTrinoPool(ctx, poolID) + if err != nil || pool == nil { + t.Fatalf("get pool: %v", err) + } + if pool.DesiredReleaseID == "stale-release" || pool.Frozen { + t.Fatalf("a superseded leader changed desired state: %+v", pool) + } + + // The current leader still writes normally. + spec.DesiredReleaseID = "current-release" + if err := store.UpsertTrinoPoolSpec(ctx, current, spec); err != nil { + t.Fatalf("current leader publication: %v", err) + } +} + +// Missing desired configuration freezes the pool. It must never be able to +// express itself as a desired count of zero. +func TestTrinoPoolRejectsAnEmptyDesiredCount(t *testing.T) { + store := newIsolatedConfigStore(t) + spec := poolSpec() + spec.DesiredInstances = 0 + if err := store.SeedTrinoPool(context.Background(), spec); err == nil { + t.Fatal("a desired count of zero was accepted") + } +} + +func TestAcquireTrinoPoolAuthorityIsMonotonic(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + + first := claimPool(t, store, "cp-a") + second := claimPool(t, store, "cp-b") + if second.Epoch <= first.Epoch { + t.Fatalf("epoch did not advance: %d -> %d", first.Epoch, second.Epoch) + } + + // The superseded leader keeps running until it notices. Its writes must be + // refused rather than silently applied on top of the new leader's. + err := store.CreateTrinoPoolInstance(ctx, first, newInstance("i-stale", trinopool.PhasePending)) + if !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader create error = %v, want ErrTrinoPoolConflict", err) + } + instances, err := store.ListTrinoPoolInstances(ctx, poolID) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(instances) != 0 { + t.Fatalf("stale leader created %d instances", len(instances)) + } + if err := store.CreateTrinoPoolInstance(ctx, second, newInstance("i-fresh", trinopool.PhasePending)); err != nil { + t.Fatalf("current leader create: %v", err) + } +} + +// Two control planes racing for the pool must not both believe they own it. +func TestConcurrentAuthorityAcquisitionYieldsDistinctEpochs(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + + const racers = 6 + epochs := make([]int64, racers) + errs := make([]error, racers) + var wg sync.WaitGroup + start := make(chan struct{}) + for index := 0; index < racers; index++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + <-start + lease, err := store.AcquireTrinoPoolAuthority(ctx, poolID, fmt.Sprintf("cp-%d", index)) + epochs[index], errs[index] = lease.Epoch, err + }(index) + } + close(start) + wg.Wait() + + seen := map[int64]bool{} + for index, err := range errs { + if err != nil { + t.Fatalf("racer %d: %v", index, err) + } + if seen[epochs[index]] { + t.Fatalf("epoch %d was handed to two owners", epochs[index]) + } + seen[epochs[index]] = true + } +} + +func TestInstancePhaseTransitionsAreCASAndValidated(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + if err := store.CreateTrinoPoolInstance(ctx, lease, newInstance("i-1", trinopool.PhasePending)); err != nil { + t.Fatalf("create: %v", err) + } + + if err := store.AdvanceTrinoPoolInstance(ctx, lease, "i-1", trinopool.PhasePending, trinopool.PhaseCreating, nil); err != nil { + t.Fatalf("advance: %v", err) + } + // Wrong expected phase: somebody else moved it first. + err := store.AdvanceTrinoPoolInstance(ctx, lease, "i-1", trinopool.PhasePending, trinopool.PhaseCreating, nil) + if !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale CAS error = %v, want ErrTrinoPoolConflict", err) + } + // Illegal transition, rejected before it reaches the database. + if err := store.AdvanceTrinoPoolInstance(ctx, lease, "i-1", trinopool.PhaseCreating, trinopool.PhaseServing, nil); err == nil { + t.Fatal("CREATING -> SERVING was accepted") + } +} + +// Retirement is irreversible. The store is the last line of defence: even a +// buggy operator must not be able to bring a retired incarnation back. +func TestRetiredInstanceCannotResume(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + if err := store.CreateTrinoPoolInstance(ctx, lease, newInstance("i-1", trinopool.PhaseSealed)); err != nil { + t.Fatalf("create: %v", err) + } + if err := store.AdvanceTrinoPoolInstance(ctx, lease, "i-1", trinopool.PhaseSealed, trinopool.PhaseRetiring, nil); err != nil { + t.Fatalf("retire: %v", err) + } + for _, target := range []trinopool.Phase{trinopool.PhaseServing, trinopool.PhaseDraining, trinopool.PhaseSealed} { + if err := store.AdvanceTrinoPoolInstance(ctx, lease, "i-1", trinopool.PhaseRetiring, target, nil); err == nil { + t.Fatalf("RETIRING -> %s was accepted", target) + } + } +} + +// An instance id is never reused, including after retirement: a recycled id +// would let a stale Kubernetes or Gateway reference resolve to a live instance. +func TestInstanceIdentitiesAreNeverReused(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + if err := store.CreateTrinoPoolInstance(ctx, lease, newInstance("i-1", trinopool.PhaseRetiring)); err != nil { + t.Fatalf("create: %v", err) + } + if err := store.AdvanceTrinoPoolInstance(ctx, lease, "i-1", trinopool.PhaseRetiring, trinopool.PhaseRetired, nil); err != nil { + t.Fatalf("retire: %v", err) + } + if err := store.CreateTrinoPoolInstance(ctx, lease, newInstance("i-1", trinopool.PhasePending)); err == nil { + t.Fatal("a retired instance id was reused") + } +} + +func TestLiveEndpointsAreUnique(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + first := newInstance("i-1", trinopool.PhasePreparing) + first.EndpointURL = "https://i-1.trino-cell-001.svc.cluster.local:8443" + if err := store.CreateTrinoPoolInstance(ctx, lease, first); err != nil { + t.Fatalf("create: %v", err) + } + second := newInstance("i-2", trinopool.PhasePreparing) + second.EndpointURL = first.EndpointURL + if err := store.CreateTrinoPoolInstance(ctx, lease, second); err == nil { + t.Fatal("two live instances shared one endpoint") + } +} + +// A lost response must be resolvable by reading the operation back under the +// same id. The same id with different content is a conflict, not a replay. +func TestOperationsAreIdempotentByIntent(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + operation := cpconfigstore.TrinoPoolOperationSpec{ + OperationID: "op-1", PoolID: poolID, InstanceID: "i-1", + Kind: cpconfigstore.TrinoPoolOperationReplace, IntentHash: "hash-a", + } + first, err := store.BeginTrinoPoolOperation(ctx, lease, operation) + if err != nil { + t.Fatalf("begin: %v", err) + } + if first.Replayed { + t.Fatal("a fresh operation reported itself as a replay") + } + replay, err := store.BeginTrinoPoolOperation(ctx, lease, operation) + if err != nil { + t.Fatalf("replay: %v", err) + } + if !replay.Replayed { + t.Fatal("an identical operation was not recognized as a replay") + } + + changed := operation + changed.IntentHash = "hash-b" + if _, err := store.BeginTrinoPoolOperation(ctx, lease, changed); !errors.Is(err, cpconfigstore.ErrTrinoPoolIntentChanged) { + t.Fatalf("changed intent error = %v, want ErrTrinoPoolIntentChanged", err) + } +} + +func TestOperationStepsAreIdempotentByPayload(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + if _, err := store.BeginTrinoPoolOperation(ctx, lease, cpconfigstore.TrinoPoolOperationSpec{ + OperationID: "op-1", PoolID: poolID, Kind: cpconfigstore.TrinoPoolOperationReplace, IntentHash: "hash-a", + }); err != nil { + t.Fatalf("begin: %v", err) + } + + recorded, err := store.RecordTrinoPoolOperationStep(ctx, lease, "op-1", "register", "payload-a", "OK", `{"instanceId":"i-1"}`) + if err != nil { + t.Fatalf("record step: %v", err) + } + if recorded.Replayed { + t.Fatal("a fresh step reported itself as a replay") + } + again, err := store.RecordTrinoPoolOperationStep(ctx, lease, "op-1", "register", "payload-a", "OK", `{"instanceId":"i-1"}`) + if err != nil { + t.Fatalf("replay step: %v", err) + } + // jsonb re-serializes, so compare the decoded documents rather than bytes. + if !again.Replayed || !sameJSON(t, again.Result, recorded.Result) { + t.Fatalf("step replay = %+v, want the recorded result %+v", again, recorded) + } + if _, err := store.RecordTrinoPoolOperationStep(ctx, lease, "op-1", "register", "payload-b", "OK", `{}`); !errors.Is(err, cpconfigstore.ErrTrinoPoolIntentChanged) { + t.Fatalf("changed step payload error = %v, want ErrTrinoPoolIntentChanged", err) + } +} + +func sameJSON(t *testing.T, left, right string) bool { + t.Helper() + var leftValue, rightValue any + if err := json.Unmarshal([]byte(left), &leftValue); err != nil { + t.Fatalf("decode %q: %v", left, err) + } + if err := json.Unmarshal([]byte(right), &rightValue); err != nil { + t.Fatalf("decode %q: %v", right, err) + } + return reflect.DeepEqual(leftValue, rightValue) +} + +// The projection watermark is the fence that stops a stale replica from serving +// a regressing authorization bundle. It only ever moves forward. +func TestProjectionWatermarkNeverRegresses(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + if err := store.AdvanceTrinoPoolProjection(ctx, lease, 5, "digest-5"); err != nil { + t.Fatalf("advance: %v", err) + } + if err := store.AdvanceTrinoPoolProjection(ctx, lease, 4, "digest-4"); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("regressing advance error = %v, want ErrTrinoPoolConflict", err) + } + projection, err := store.GetTrinoPoolProjection(ctx, poolID) + if err != nil { + t.Fatalf("get projection: %v", err) + } + if projection.AcceptedRevision != 5 || projection.AcceptedDigest != "digest-5" { + t.Fatalf("projection = %+v, want revision 5", projection) + } + + // A superseded leader cannot move the watermark at all. + stale := lease + stale.Epoch-- + if err := store.AdvanceTrinoPoolProjection(ctx, stale, 6, "digest-6"); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader advance error = %v, want ErrTrinoPoolConflict", err) + } +} + +// Desired, published and admitted are three separate facts. A tenant that is +// enabled but not yet admitted must be distinguishable from one that is live. +func TestPublicationTracksDesiredAndAdmittedSeparately(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + if err := store.SetTrinoPoolPublicationDesired(ctx, lease, poolID, "org-a", 7); err != nil { + t.Fatalf("set desired: %v", err) + } + publication, err := store.GetTrinoPoolPublication(ctx, poolID, "org-a") + if err != nil || publication == nil { + t.Fatalf("get publication: %v", err) + } + if publication.DesiredRevision != 7 || publication.AdmittedRevision != 0 || publication.State != cpconfigstore.TrinoPublicationPending { + t.Fatalf("publication = %+v, want a pending tenant at desired revision 7", publication) + } + + if err := store.RecordTrinoPoolPublicationAdmitted(ctx, lease, poolID, "org-a", 7, "pub-1", `{"phase":"ADMITTED"}`); err != nil { + t.Fatalf("record admitted: %v", err) + } + publication, err = store.GetTrinoPoolPublication(ctx, poolID, "org-a") + if err != nil || publication == nil { + t.Fatalf("get publication: %v", err) + } + if publication.AdmittedRevision != 7 || publication.State != cpconfigstore.TrinoPublicationAdmitted { + t.Fatalf("publication = %+v, want an admitted tenant", publication) + } +} + +// A step is recorded UNKNOWN before the external effect and re-recorded with +// the outcome after it. If the second call returned the stored row unchanged, +// a step could never leave UNKNOWN: the cross-leader read-back that keys on a +// completed step would be unreachable and every retry would repeat the call. +func TestOperationStepOutcomeAdvancesOutOfUnknown(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + if _, err := store.BeginTrinoPoolOperation(ctx, lease, cpconfigstore.TrinoPoolOperationSpec{ + OperationID: "op-1", PoolID: poolID, Kind: cpconfigstore.TrinoPoolOperationReplace, IntentHash: "hash-a", + }); err != nil { + t.Fatalf("begin: %v", err) + } + + if _, err := store.RecordTrinoPoolOperationStep(ctx, lease, "op-1", "admit", "payload-a", + cpconfigstore.TrinoPoolStepOutcomeUnknown, "{}"); err != nil { + t.Fatalf("record intent: %v", err) + } + recorded, err := store.RecordTrinoPoolOperationStep(ctx, lease, "op-1", "admit", "payload-a", + cpconfigstore.TrinoPoolStepOutcomeOK, `{"phase":"ACTIVE"}`) + if err != nil { + t.Fatalf("record outcome: %v", err) + } + if recorded.Outcome != cpconfigstore.TrinoPoolStepOutcomeOK || !recorded.Replayed { + t.Fatalf("step = %+v, want a replayed step recorded OK", recorded) + } + + // A DECIDED outcome is never re-decided: rewriting it is exactly the loss of + // history the journal exists to prevent. + again, err := store.RecordTrinoPoolOperationStep(ctx, lease, "op-1", "admit", "payload-a", + cpconfigstore.TrinoPoolStepOutcomeFailed, `{"phase":"REFUSED"}`) + if err != nil { + t.Fatalf("re-record: %v", err) + } + if again.Outcome != cpconfigstore.TrinoPoolStepOutcomeOK { + t.Fatalf("outcome = %q, want the recorded OK to stand", again.Outcome) + } +} + +// The candidate gate compares against the pool's published catalog revision, so +// that revision has to be written by whatever publishes catalogs. It only moves +// forward: two publications can report out of order, and moving the gate +// backwards would certify a coordinator missing the newest tenant. +func TestPublicationRevisionOnlyAdvances(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + if err := store.RecordTrinoPoolPublicationRevision(ctx, lease, poolID, 7); err != nil { + t.Fatalf("record revision: %v", err) + } + if err := store.RecordTrinoPoolPublicationRevision(ctx, lease, poolID, 5); err != nil { + t.Fatalf("record older revision: %v", err) + } + pool, err := store.GetTrinoPool(ctx, poolID) + if err != nil || pool == nil { + t.Fatalf("get pool: %v", err) + } + if pool.PublicationRevision != 7 { + t.Fatalf("publication revision = %d, want the highest published (7)", pool.PublicationRevision) + } + + stale := lease + stale.Epoch-- + if err := store.RecordTrinoPoolPublicationRevision(ctx, stale, poolID, 9); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader error = %v, want ErrTrinoPoolConflict", err) + } +} + +// A desired generation that went backwards is a CONFIGURATION problem, not a +// lost fence. Reporting it as a conflict ended the leadership term on every +// tick and handed the pool to a replica that did the same. +func TestBackwardsGenerationIsNotAFenceConflict(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + spec := poolSpec() + spec.Generation = 5 + if err := store.UpsertTrinoPoolSpec(ctx, lease, spec); err != nil { + t.Fatalf("publish generation 5: %v", err) + } + spec.Generation = 4 + err := store.UpsertTrinoPoolSpec(ctx, lease, spec) + if !errors.Is(err, cpconfigstore.ErrTrinoPoolStaleGeneration) { + t.Fatalf("backwards generation error = %v, want ErrTrinoPoolStaleGeneration", err) + } + if errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatal("a stale generation was reported as a lost fence") + } +} + +// The publication barrier's record is what a NEW leader reads instead of its +// own memory: which binding was published, which barrier is open, and which +// target actually committed. +func TestTenantPublicationRecordsBindingAndAdmission(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + if err := store.RecordTrinoPoolTenantPrincipals(ctx, lease, poolID, "org-a", "binding-1"); err != nil { + t.Fatalf("record principals: %v", err) + } + if err := store.RecordTrinoPoolPublicationOpen(ctx, lease, poolID, "org-a", "pub-1", "c7.pabc"); err != nil { + t.Fatalf("record open: %v", err) + } + if err := store.RecordTrinoPoolPublicationCommitted(ctx, lease, poolID, "org-a", "c7.pabc", `{"receipts":3}`); err != nil { + t.Fatalf("record commit: %v", err) + } + + publications, err := store.ListTrinoPoolPublications(ctx, poolID) + if err != nil || len(publications) != 1 { + t.Fatalf("list publications: %v (%d rows)", err, len(publications)) + } + publication := publications[0] + if publication.PrincipalRevision != "binding-1" || publication.AdmittedTargetRevision != "c7.pabc" || + publication.State != cpconfigstore.TrinoPublicationAdmitted { + t.Fatalf("publication = %+v, want an admitted tenant at the committed target", publication) + } + // The committed barrier stops being the LIVE one. Leaving it named here + // would make the driver select a finished publication as the attempt in + // flight forever, and the Gateway never retracts an opened admission gate. + if publication.PublicationID != "" || publication.TargetRevision != "" { + t.Fatalf("publication = %+v, want the finished barrier cleared", publication) + } + + // Revocation KEEPS the row. Deleting it would read as "never published", and + // the next tick would republish the binding of a tenant meant to be gone. + if err := store.RecordTrinoPoolTenantRevoked(ctx, lease, poolID, "org-a", "warehouse removed"); err != nil { + t.Fatalf("record revocation: %v", err) + } + publications, err = store.ListTrinoPoolPublications(ctx, poolID) + if err != nil || len(publications) != 1 { + t.Fatalf("list after revocation: %v (%d rows)", err, len(publications)) + } + if publications[0].State != cpconfigstore.TrinoPublicationRevoked || + publications[0].AdmittedTargetRevision != "" { + t.Fatalf("publication = %+v, want a revoked tenant with no admitted target", publications[0]) + } + + stale := lease + stale.Epoch-- + if err := store.RecordTrinoPoolTenantPrincipals(ctx, stale, poolID, "org-a", "binding-2"); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader error = %v, want ErrTrinoPoolConflict", err) + } +} + +// A new barrier does not retract an admission, and clearing a dead one does not +// either. +// +// Both matter outside the driver: an operator surface keyed on the state would +// otherwise flap a serving warehouse back to Provisioning every time one of its +// logins changed, and an attempt that was abandoned has to leave the durable +// record without taking the tenant's admission with it. +func TestPublicationBarrierLivenessIsSeparateFromAdmission(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + if err := store.RecordTrinoPoolTenantPrincipals(ctx, lease, poolID, "org-a", "binding-1"); err != nil { + t.Fatalf("record principals: %v", err) + } + // A tenant that has never been admitted reports that it is being admitted. + if err := store.RecordTrinoPoolPublicationOpen(ctx, lease, poolID, "org-a", "pub-1", "b1.a1"); err != nil { + t.Fatalf("record open: %v", err) + } + if got := onePublication(t, store, "org-a"); got.State != cpconfigstore.TrinoPublicationAdmitting { + t.Fatalf("state = %q, want admitting for a tenant that was never admitted", got.State) + } + if err := store.RecordTrinoPoolPublicationCommitted(ctx, lease, poolID, "org-a", "b1.a1", `{"receipts":3}`); err != nil { + t.Fatalf("record commit: %v", err) + } + + // Its next barrier - a new login - leaves the admission alone. + if err := store.RecordTrinoPoolTenantPrincipals(ctx, lease, poolID, "org-a", "binding-2"); err != nil { + t.Fatalf("record second principals: %v", err) + } + if err := store.RecordTrinoPoolPublicationOpen(ctx, lease, poolID, "org-a", "pub-2", "b2.a2"); err != nil { + t.Fatalf("record second open: %v", err) + } + got := onePublication(t, store, "org-a") + if got.State != cpconfigstore.TrinoPublicationAdmitted || got.AdmittedTargetRevision != "b1.a1" { + t.Fatalf("publication = %+v, want the serving tenant to stay admitted at its previous target", got) + } + if got.PublicationID != "pub-2" { + t.Fatalf("live barrier = %q, want pub-2", got.PublicationID) + } + + // Clearing the attempt takes the barrier, not the admission. + if err := store.ClearTrinoPoolPublicationBarrier(ctx, lease, poolID, "org-a"); err != nil { + t.Fatalf("clear barrier: %v", err) + } + got = onePublication(t, store, "org-a") + if got.PublicationID != "" || got.TargetRevision != "" { + t.Fatalf("publication = %+v, want no live barrier", got) + } + if got.State != cpconfigstore.TrinoPublicationAdmitted || got.AdmittedTargetRevision != "b1.a1" { + t.Fatalf("publication = %+v, want the admission preserved", got) + } + + // A tenant that was never admitted falls back to whether its binding was + // published at all. + if err := store.RecordTrinoPoolTenantPrincipals(ctx, lease, poolID, "org-b", "binding-1"); err != nil { + t.Fatalf("record principals for org-b: %v", err) + } + if err := store.RecordTrinoPoolPublicationOpen(ctx, lease, poolID, "org-b", "pub-3", "b1.a1"); err != nil { + t.Fatalf("record open for org-b: %v", err) + } + if err := store.ClearTrinoPoolPublicationBarrier(ctx, lease, poolID, "org-b"); err != nil { + t.Fatalf("clear barrier for org-b: %v", err) + } + if got := onePublication(t, store, "org-b"); got.State != cpconfigstore.TrinoPublicationPublished { + t.Fatalf("state = %q, want published for a tenant whose binding is current but was never admitted", got.State) + } + + stale := lease + stale.Epoch-- + if err := store.ClearTrinoPoolPublicationBarrier(ctx, stale, poolID, "org-a"); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader error = %v, want ErrTrinoPoolConflict", err) + } +} + +func onePublication(t *testing.T, store *cpconfigstore.ConfigStore, orgID string) cpconfigstore.TrinoPoolPublication { + t.Helper() + publications, err := store.ListTrinoPoolPublications(context.Background(), poolID) + if err != nil { + t.Fatalf("list publications: %v", err) + } + for _, publication := range publications { + if publication.OrgID == orgID { + return publication + } + } + t.Fatalf("no publication row for %s", orgID) + return cpconfigstore.TrinoPoolPublication{} +} + +// A tenant's occurrence counter is what makes its NEXT barrier - or its next +// revocation - a new operation. Sharing an identity with the previous one would +// replay that one's recorded outcome, which for a revocation means a tenant +// nobody revoked stays admitted. + +// An occurrence that stands for a request in flight is durable, and only a +// DEFINITE outcome closes it. +// +// A lost response leaves the request possibly still executing at the Gateway, +// so the next pass has to reissue that exact step rather than mint a new one - +// which it can only do if what the occurrence stands for survived the restart +// that may have happened in between. +func TestPendingPublicationIntentIsDurable(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + body := `{"revision":"binding-1","principals":["acme","acme.analyst"]}` + first, err := store.BeginTrinoPoolPublicationIntent(ctx, lease, poolID, "org-a", + cpconfigstore.TrinoPublicationIntentPrincipals, body) + if err != nil || first != 1 { + t.Fatalf("begin intent = %d, %v; want occurrence 1", first, err) + } + got := onePublication(t, store, "org-a") + if got.PendingIntent != cpconfigstore.TrinoPublicationIntentPrincipals { + t.Fatalf("pending intent = %q, want the publication it stands for", got.PendingIntent) + } + // The body survives, so the reissue can be byte-identical - including for a + // tenant whose desired binding has since changed or gone. + var stored struct { + Revision string `json:"revision"` + Principals []string `json:"principals"` + } + if err := json.Unmarshal([]byte(got.PendingPayload), &stored); err != nil { + t.Fatalf("stored request %q is not readable: %v", got.PendingPayload, err) + } + if stored.Revision != "binding-1" || !reflect.DeepEqual(stored.Principals, []string{"acme", "acme.analyst"}) { + t.Fatalf("stored request = %+v, want the exact request that was issued", stored) + } + + // The Gateway answered: the checkpoint closes the occurrence in the same + // write, so a crash cannot leave a checkpointed binding with an occurrence + // still claiming to be in flight. + if err := store.RecordTrinoPoolTenantPrincipals(ctx, lease, poolID, "org-a", "binding-1"); err != nil { + t.Fatalf("record principals: %v", err) + } + if got := onePublication(t, store, "org-a"); got.PendingIntent != "" || got.PendingPayload != "{}" || + got.PrincipalRevision != "binding-1" { + t.Fatalf("publication = %+v, want a checkpointed binding and no open occurrence", got) + } + + // A revocation takes its own occurrence, and a definite refusal closes it + // without moving the checkpoint. + second, err := store.BeginTrinoPoolPublicationIntent(ctx, lease, poolID, "org-a", + cpconfigstore.TrinoPublicationIntentRevoke, `{"reason":"the warehouse is no longer served by this pool"}`) + if err != nil || second != 2 { + t.Fatalf("begin revoke intent = %d, %v; want occurrence 2", second, err) + } + if err := store.ResolveTrinoPoolPublicationIntent(ctx, lease, poolID, "org-a"); err != nil { + t.Fatalf("resolve intent: %v", err) + } + got = onePublication(t, store, "org-a") + if got.PendingIntent != "" || got.PendingPayload != "{}" || got.Attempt != 2 || + got.PrincipalRevision != "binding-1" { + t.Fatalf("publication = %+v, want the occurrence closed and the checkpoint untouched", got) + } + + // A revocation clears it too, so a revoked tenant never looks like one with + // a request outstanding. + if _, err := store.BeginTrinoPoolPublicationIntent(ctx, lease, poolID, "org-a", + cpconfigstore.TrinoPublicationIntentRevoke, `{"reason":"gone"}`); err != nil { + t.Fatalf("begin second revoke intent: %v", err) + } + if err := store.RecordTrinoPoolTenantRevoked(ctx, lease, poolID, "org-a", "warehouse removed"); err != nil { + t.Fatalf("record revocation: %v", err) + } + if got := onePublication(t, store, "org-a"); got.PendingIntent != "" || got.PendingPayload != "{}" { + t.Fatalf("publication = %+v, want no open occurrence after a revocation", got) + } + + if _, err := store.BeginTrinoPoolPublicationIntent(ctx, lease, poolID, "org-a", "something-else", body); err == nil { + t.Fatal("an unknown intent kind was accepted") + } + // An occurrence with no body could not be replayed, which is the whole + // point of recording it. + if _, err := store.BeginTrinoPoolPublicationIntent(ctx, lease, poolID, "org-a", + cpconfigstore.TrinoPublicationIntentPrincipals, ""); err == nil { + t.Fatal("an intent with no request body was accepted") + } + stale := lease + stale.Epoch-- + if _, err := store.BeginTrinoPoolPublicationIntent(ctx, stale, poolID, "org-a", + cpconfigstore.TrinoPublicationIntentPrincipals, body); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader error = %v, want ErrTrinoPoolConflict", err) + } +} +func TestPublicationAttemptsAreMonotonePerTenant(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + first, err := store.BeginTrinoPoolPublicationAttempt(ctx, lease, poolID, "org-a") + if err != nil { + t.Fatalf("begin attempt: %v", err) + } + second, err := store.BeginTrinoPoolPublicationAttempt(ctx, lease, poolID, "org-a") + if err != nil { + t.Fatalf("begin second attempt: %v", err) + } + if first != 1 || second != 2 { + t.Fatalf("attempts = %d then %d, want 1 then 2", first, second) + } + // Another tenant counts independently. + other, err := store.BeginTrinoPoolPublicationAttempt(ctx, lease, poolID, "org-b") + if err != nil { + t.Fatalf("begin attempt for another tenant: %v", err) + } + if other != 1 { + t.Fatalf("attempt for a second tenant = %d, want its own 1", other) + } + + stale := lease + stale.Epoch-- + if _, err := store.BeginTrinoPoolPublicationAttempt(ctx, stale, poolID, "org-a"); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader error = %v, want ErrTrinoPoolConflict", err) + } +} + +// One unserviceable tenant must not busy-loop or starve the tenants behind it: +// the driver takes one tenant per tick, so the wait a failure earns has to be +// durable and per tenant. +func TestPublicationFailureRecordsADurableWait(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + next := time.Now().UTC().Add(45 * time.Second) + if err := store.RecordTrinoPoolPublicationFailure(ctx, lease, poolID, "org-a", next, "principal conflict"); err != nil { + t.Fatalf("record failure: %v", err) + } + if err := store.RecordTrinoPoolPublicationFailure(ctx, lease, poolID, "org-a", next, "principal conflict"); err != nil { + t.Fatalf("record second failure: %v", err) + } + publication, err := store.GetTrinoPoolPublication(ctx, poolID, "org-a") + if err != nil || publication == nil { + t.Fatalf("get publication: %v", err) + } + if publication.Attempts != 2 || publication.NextAttemptAt == nil || publication.LastError == "" { + t.Fatalf("publication = %+v, want two recorded attempts and a next attempt time", publication) + } + + if err := store.ClearTrinoPoolPublicationFailure(ctx, lease, poolID, "org-a"); err != nil { + t.Fatalf("clear failure: %v", err) + } + publication, err = store.GetTrinoPoolPublication(ctx, poolID, "org-a") + if err != nil || publication == nil { + t.Fatalf("get publication: %v", err) + } + if publication.Attempts != 0 || publication.NextAttemptAt != nil { + t.Fatalf("publication = %+v, want the backoff cleared after a step that worked", publication) + } +} + +// The projection fence needs an ORDER, not just a fingerprint: every control +// plane builds the projection from its own view, so a replica that is behind +// cannot tell that it is from a digest alone. The authority assigns it. +func TestAcceptedProjectionIsOrderedAndStableForUnchangedContent(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + first, err := store.AcceptTrinoPoolProjection(ctx, lease, "digest-1") + if err != nil { + t.Fatalf("accept: %v", err) + } + same, err := store.AcceptTrinoPoolProjection(ctx, lease, "digest-1") + if err != nil { + t.Fatalf("re-accept: %v", err) + } + if first != 1 || same != first { + t.Fatalf("revisions = %d then %d, want an unchanged projection to keep its revision", first, same) + } + next, err := store.AcceptTrinoPoolProjection(ctx, lease, "digest-2") + if err != nil { + t.Fatalf("accept a changed projection: %v", err) + } + if next != first+1 { + t.Fatalf("revision = %d, want %d", next, first+1) + } + + projection, err := store.GetTrinoPoolProjection(ctx, poolID) + if err != nil { + t.Fatalf("read projection: %v", err) + } + if projection.AcceptedDigest != "digest-2" || projection.AcceptedRevision != next { + t.Fatalf("projection = %+v, want the accepted digest at revision %d", projection, next) + } + + // A superseded leader cannot move it - which is the case that matters: a + // delayed write from the previous authority must not reinstate an older + // projection after a newer one is in effect. + stale := lease + stale.Epoch-- + if _, err := store.AcceptTrinoPoolProjection(ctx, stale, "digest-old"); !errors.Is(err, cpconfigstore.ErrTrinoPoolConflict) { + t.Fatalf("stale leader error = %v, want ErrTrinoPoolConflict", err) + } + projection, err = store.GetTrinoPoolProjection(ctx, poolID) + if err != nil { + t.Fatalf("re-read projection: %v", err) + } + if projection.AcceptedDigest != "digest-2" { + t.Fatalf("a superseded leader changed the accepted projection: %+v", projection) + } +} + +// The projection a pooled cell accepts must describe ONE state of the database. +// +// The pool row lock serializes acceptances against each other, but nothing +// stops the org/user/team writers - so under the default isolation the separate +// reads this builds from could straddle such a write and be accepted as a +// coherent projection that never existed. +func TestAcceptedProjectionReadsOneSnapshotOfItsSources(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + seedTrinoOrg(t, store, "acme") + if err := store.EnableTrino("acme", cpconfigstore.TrinoSettings{}); err != nil { + t.Fatalf("EnableTrino: %v", err) + } + + // A writer commits a NEW tenant while the builder is between its reads. + var seenFirst, seenSecond int + _, _, err := store.AcceptTrinoPoolProjectionFrom(ctx, lease, func(sources cpconfigstore.TrinoProjectionSources) (string, error) { + seenFirst = len(sources.Orgs) + // This commits in another session, after the transaction's snapshot was + // taken. A repeatable-read transaction must not see it. + seedTrinoOrg(t, store, "beta") + if err := store.EnableTrino("beta", cpconfigstore.TrinoSettings{}); err != nil { + return "", err + } + again, err := sources.Reread() + if err != nil { + return "", err + } + seenSecond = len(again) + return "digest-1", nil + }) + if err != nil { + t.Fatalf("accept: %v", err) + } + if seenFirst != 1 { + t.Fatalf("the builder saw %d orgs, want the one that existed", seenFirst) + } + if seenSecond != seenFirst { + t.Fatalf("a second read inside the same acceptance saw %d orgs after %d: the projection is built from two different states", + seenSecond, seenFirst) + } +} + +// A project-scoped login's policy must come from the SAME read as the rest of +// the projection. +// +// The snapshot-backed resolver is refreshed on a poll, so a scope taken from it +// can be older than the rows the acceptance transaction just read - and the +// result would be accepted as one coherent projection. Here the team is +// disabled in the database and the snapshot is deliberately NOT reloaded: the +// accepted projection must reflect the database. +func TestAcceptedProjectionDerivesScopesFromItsOwnRead(t *testing.T) { + ctx := context.Background() + store := newPoolStore(t) + lease := claimPool(t, store, "cp-a") + + seedTrinoOrg(t, store, "acme") + if err := store.EnableTrino("acme", cpconfigstore.TrinoSettings{}); err != nil { + t.Fatalf("EnableTrino: %v", err) + } + if _, err := cpconfigstore.UpsertOrgTeamTx(store.DB(), "acme", cpconfigstore.OrgTeamUpsert{ + TeamID: 7, SchemaName: "posthog_7", + }); err != nil { + t.Fatalf("UpsertOrgTeamTx: %v", err) + } + if err := store.CreateOrgUser("acme", "posthog_team_7", "$2a$10$team7"); err != nil { + t.Fatalf("CreateOrgUser: %v", err) + } + if err := store.DB().Exec( + `UPDATE duckgres_org_users SET access_mode = 'project_reader', team_id = 7 + WHERE org_id = 'acme' AND username = 'posthog_team_7'`).Error; err != nil { + t.Fatalf("bind the project login: %v", err) + } + if err := store.ReloadSnapshot(); err != nil { + t.Fatalf("ReloadSnapshot: %v", err) + } + + // The team is disabled in the database. Nothing reloads the snapshot, so + // the cache still reports the login as scoped to an enabled team. + if err := store.DB().Exec( + `UPDATE duckgres_org_teams SET enabled = false WHERE org_id = 'acme' AND team_id = 7`).Error; err != nil { + t.Fatalf("disable the team: %v", err) + } + + var scoped *cpconfigstore.TrinoOrgUser + if _, _, err := store.AcceptTrinoPoolProjectionWith(ctx, lease, func(orgs []cpconfigstore.TrinoEnabledOrg) (string, error) { + for i := range orgs { + for j := range orgs[i].Users { + if orgs[i].Users[j].Username == "posthog_team_7" { + scoped = &orgs[i].Users[j] + } + } + } + return "digest-1", nil + }); err != nil { + t.Fatalf("accept: %v", err) + } + if scoped == nil || scoped.Scope == nil { + t.Fatalf("the project login was dropped or unscoped: %+v", scoped) + } + if len(scoped.Scope.AllowedSchemas) != 0 || !scoped.Scope.ReadOnly { + t.Fatalf("scope = %+v, want the fail-closed policy the DISABLED team implies, not the cached one", + *scoped.Scope) + } + + // The cache, unreloaded, still reports the old policy - which is exactly + // why the projection must not be built from it. + cached, ok := store.OrgUserQueryAccess("acme", "posthog_team_7") + if !ok || len(cached.AllowedSchemas) == 0 { + t.Fatalf("the snapshot cache no longer holds the stale policy (%+v, ok=%v); this test proves nothing", cached, ok) + } +} diff --git a/tests/controlplane/controlplane_test.go b/tests/controlplane/controlplane_test.go index 784349dc3..4ebbce329 100644 --- a/tests/controlplane/controlplane_test.go +++ b/tests/controlplane/controlplane_test.go @@ -18,7 +18,7 @@ import ( "testing" "time" - _ "github.com/lib/pq" + "github.com/lib/pq" "github.com/posthog/duckgres/server" ) @@ -207,13 +207,19 @@ users: return h } -func (h *cpHarness) openConn(t *testing.T) *sql.DB { +func (h *cpHarness) openConn(t *testing.T, dialers ...pq.Dialer) *sql.DB { t.Helper() - dsn := fmt.Sprintf("host=127.0.0.1 port=%d user=testuser password=testpass sslmode=require connect_timeout=10", h.port) - db, err := sql.Open("postgres", dsn) + // lib/pq applies connect_timeout through ReadyForQuery, including worker initialization. + // Allow the default session initialization timeout; lib/pq clears it before queries. + dsn := fmt.Sprintf("host=127.0.0.1 port=%d user=testuser password=testpass sslmode=require connect_timeout=%d", h.port, int(server.DefaultSessionInitTimeout/time.Second)) + connector, err := pq.NewConnector(dsn) if err != nil { t.Fatalf("Failed to open connection: %v", err) } + if len(dialers) > 0 { + connector.Dialer(dialers[0]) + } + db := sql.OpenDB(connector) db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) t.Cleanup(func() { _ = db.Close() }) @@ -240,6 +246,11 @@ func (h *cpHarness) waitForLog(substr string, timeout time.Duration) error { func (h *cpHarness) cleanup(t *testing.T) { t.Helper() + defer func() { + if t.Failed() { + t.Logf("Control plane logs:\n%s", h.logBuf.String()) + } + }() if h.cmd.Process == nil { return @@ -318,6 +329,66 @@ func TestControlPlaneBasic(t *testing.T) { } } +func TestControlPlaneConnectionWaitsForDelayedStartup(t *testing.T) { + h := startControlPlane(t, defaultOpts()) + if err := h.sendSignal(syscall.SIGSTOP); err != nil { + t.Fatalf("Pause control plane: %v", err) + } + dialer := &delayedStartupDialer{connected: make(chan struct{})} + stopDelay := make(chan struct{}) + resumeErr := make(chan error, 1) + go func() { + select { + case <-dialer.connected: + delay := time.NewTimer(11 * time.Second) + defer delay.Stop() + select { + case <-delay.C: + case <-stopDelay: + } + case <-stopDelay: + } + resumeErr <- h.sendSignal(syscall.SIGCONT) + }() + defer func() { + close(stopDelay) + if err := <-resumeErr; err != nil { + t.Errorf("Resume control plane: %v", err) + } + }() + + db := h.openConn(t, dialer) + ctx, cancel := context.WithTimeout(context.Background(), server.DefaultSessionInitTimeout) + defer cancel() + var result int + if err := db.QueryRowContext(ctx, "SELECT 1").Scan(&result); err != nil { + t.Fatalf("Delayed startup query failed: %v\nLogs:\n%s", err, h.logBuf.String()) + } + if result != 1 { + t.Fatalf("Expected 1, got %d", result) + } + if attempts := strings.Count(h.logBuf.String(), "Connection accepted."); attempts != 1 { + t.Fatalf("Delayed startup used %d connection attempts, want 1", attempts) + } +} + +type delayedStartupDialer struct { + connected chan struct{} + once sync.Once +} + +func (d *delayedStartupDialer) Dial(network, address string) (net.Conn, error) { + return d.DialTimeout(network, address, 0) +} + +func (d *delayedStartupDialer) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { + conn, err := net.DialTimeout(network, address, timeout) + if err == nil { + d.once.Do(func() { close(d.connected) }) + } + return conn, err +} + func TestHandoverPreservesActiveQuery(t *testing.T) { h := startControlPlane(t, defaultOpts()) @@ -732,11 +803,9 @@ func TestUpgradeWithMaxWorkers(t *testing.T) { h.doHandover(t) - // Verify new connections work after upgrade. The first post-handover - // query has to spawn and DuckDB-pre-warm a fresh worker process; on slow - // CI runners that easily exceeds the default lib/pq read deadline. Wrap - // the query in an explicit 60s context so we wait long enough for the - // worker to come up rather than racing the warmup. + // Verify new connections work after upgrade. openConn bounds the handshake, + // which includes starting and pre-warming a worker. The context also bounds + // query execution; it cannot extend lib/pq's connect_timeout deadline. db := h.openConn(t) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() diff --git a/tests/mw-dev/README.md b/tests/mw-dev/README.md index e7ab09a45..db00c2636 100644 --- a/tests/mw-dev/README.md +++ b/tests/mw-dev/README.md @@ -171,6 +171,11 @@ store; upstream `trinodb/trino` is not compatible. Update the default in `run.sh` and `e2e-mw-dev.yml` together when promoting the regular E2E Trino build. The frozen benchmark retains its separate pin in `scenario-dev.yml` until its independent migration. +On statement failure, the harness reports the query ID, error codes and a bounded +exception-class chain alongside the existing top-level message. It excludes nested +messages, stack traces and response URLs because these can contain credentials or +internal infrastructure details. A failure still stops the test without retrying +the statement; these diagnostics do not classify a storage failure as transient. The suite asserts per-user logins on every run: an org user authenticates as `.` with its pgwire password, reads only its own org's catalog, is attributed to its org in the admin query list, and stops diff --git a/tests/mw-dev/e2e/harness.sh b/tests/mw-dev/e2e/harness.sh index 176f8620e..20c12ec9b 100755 --- a/tests/mw-dev/e2e/harness.sh +++ b/tests/mw-dev/e2e/harness.sh @@ -946,6 +946,200 @@ compute_usage_pull_api() { # org password # the janitor's cap sweep (5s tick + config-poll snapshot reload — the poll # below covers both), retiring the OLDEST excess; restoring 0 (unlimited) # afterwards leaves the remaining parked worker alone. +# The shared Trino compute pool ships DISABLED. What this asserts is exactly +# that: with no pooled cell configured, the control plane creates no pool +# workload at all. A feature that is supposed to change nothing is only +# credible if "nothing changed" is actually checked. +# +# What CANNOT be asserted in-Job, and why: +# +# - The durable pool tables. This Job reaches duckgres over pgwire and the +# admin API; it has no config-store credential, so a row-level assertion is +# not available here. The migration and every fenced write are covered by +# tests/configstore/trino_pool_postgres_test.go against a real PostgreSQL. +# - The serving path (create -> admit -> drain -> retire). It needs a registry +# entry with mode: "shared-pool", a Golden-Chart blueprint artifact with a +# real image digest, a Gateway running the pooled protocol, and the pool +# feature flags on - a configuration this Job does not provision. That path +# is covered by the controlplane/ unit tests (fake clientset + fake Gateway) +# and the real-PostgreSQL publisher tests, and stays UNVERIFIED against a +# live cluster until an authorized deployment enables the flags. +# - The catalog-watermark admission gate (a catalog that committed while its +# revision checkpoint failed must not let a tenant be admitted against the +# older revision). It needs the pooled catalog writer, which publishes to +# the Trino-side catalog store this Job holds no credential for. Covered by +# TestAdmissionStaysClosedUntilTheCatalogWatermarkIsKnown (the operator +# loop) and the real-PostgreSQL TestPooledCatalogWriter* cases (the store +# side, as the scoped publisher role). +# - The member retry paths: a Gateway response lost in transit, and the +# failure repair of a coordinator that restarted in place. Both need the +# serving path above, plus fault injection this Job cannot perform (drop one +# specific Gateway response; end one specific container mid-flight). They are +# covered by controlplane/trino_pool_member_retries_test.go and stay +# UNVERIFIED against a live Gateway. +trino_shared_pool_disabled() { + log "shared Trino pool: asserting the feature is inert" + + # Every object a pooled instance owns carries this label, so the check holds + # regardless of how the objects are named. + selector="app.kubernetes.io/managed-by=duckgres-trino-pool" + + # Each count is fail-closed: an API permission problem or an unavailable + # apiserver must FAIL this assertion, never pass it. Defaulting an error to + # zero would turn "I could not look" into "there is nothing there", which is + # the one answer this check is supposed to earn. + for kind in pods deployments services; do + found="$(pool_object_count "$kind" "$selector")" + [ "$found" = "0" ] || fail "shared pool: $found pooled $kind exist with the feature disabled" + done + + log "shared pool OK: no pooled workload exists (durable state + serving path are unit/PG-tested only)" +} + +# pool_object_count counts objects of one kind, or FAILS. Both the API call and +# the projection have to succeed for the number to mean anything. +pool_object_count() { # kind selector + body="$(kubectl get "$1" -A -l "$2" -o json)" \ + || fail "shared pool: could not list $1 (a failed observation is not evidence of absence)" + count="$(printf %s "$body" | jq -r '.items | length')" \ + || fail "shared pool: could not read the $1 listing" + printf %s "$count" +} + +# The ACTIVE path, for a cluster that has the pool enabled. +# +# It runs only when E2E_TRINO_POOL=1, because it needs a configuration this lane +# does not provision by default: a registry entry with mode "shared-pool", a +# blueprint artifact carrying a real image digest, a Gateway speaking the pooled +# protocol, and the feature flags on. When those exist this is the acceptance +# check, in three +# separately reported stages, because each is evidence for less than the next: +# +# [structure] ready, non-terminating coordinator pods, each with its own +# Service and its own workers. A ready pod routes nothing. +# [admission] the warehouse reports ready, which with the Gateway gate on +# means its publication committed on every serving member. +# [query] a statement run with an EXISTING login of that org returns its +# result. Needs caller-supplied credentials; skipped, and said to +# be skipped, when they are not given. No canary warehouse and no +# new secret is introduced for it. +trino_shared_pool_active() { + [ "${E2E_TRINO_POOL:-0}" = "1" ] || { + log "SKIP shared-pool active path (set E2E_TRINO_POOL=1 on a pool-enabled cluster)" + return 0 + } + pool="${E2E_TRINO_POOL_ID:-cell-001}" + want="${E2E_TRINO_POOL_MIN_SERVING:-3}" + log "shared Trino pool [structure]: waiting for $want ready coordinator instances in $pool" + + selector="app.kubernetes.io/managed-by=duckgres-trino-pool,posthog.com/trino-pool=$pool" + a=0 ready=0 + while [ "$a" -lt 60 ]; do + # Ready, NON-TERMINATING coordinators, counted from the pods themselves: the + # operator's own view is what is under test, so it cannot also be the + # evidence. A terminating pod still reports Ready for its whole grace + # period, and counting it would claim capacity that is on its way out. + # + # This is a STRUCTURAL count only. A ready coordinator pod is not an + # admitted Gateway member and routes nothing by itself; admission is + # asserted separately below. + body="$(kubectl get pods -A -l "$selector,app.kubernetes.io/component=coordinator" -o json)" \ + || fail "shared pool: could not list coordinator pods" + ready="$(printf %s "$body" | jq -r '[.items[] + | select(.metadata.deletionTimestamp == null) + | select(.status.phase=="Running") + | select([.status.conditions[]? | select(.type=="Ready" and .status=="True")] | length > 0)] | length')" \ + || fail "shared pool: could not read the coordinator listing" + [ "${ready:-0}" -ge "$want" ] && break + sleep 10; a=$((a + 1)) + done + [ "${ready:-0}" -ge "$want" ] || fail "shared pool: only $ready serving coordinator(s), want $want" + + # Each instance must have its OWN Service and its own workers: a shared + # Service or a shared discovery URI would silently merge two clusters, which + # no pod-count assertion would notice. + services="$(pool_object_count services "$selector")" + [ "${services:-0}" -ge "$want" ] || fail "shared pool: $services service(s) for $ready instance(s)" + instances="$(printf %s "$body" | jq -r '[.items[] + | select(.metadata.deletionTimestamp == null) + | .metadata.labels["posthog.com/trino-instance"]] | unique | length')" \ + || fail "shared pool: could not read coordinator instance labels" + [ "${instances:-0}" -ge "$want" ] || fail "shared pool: $instances distinct instance(s) among the coordinators" + + # Every worker must belong to an instance that has a coordinator: an orphaned + # worker set is the visible symptom of a half-retired instance. + workers="$(kubectl get pods -A -l "$selector,app.kubernetes.io/component=worker" -o json)" \ + || fail "shared pool: could not list worker pods" + known="$(printf %s "$body" | jq -c '[.items[].metadata.labels["posthog.com/trino-instance"]]')" \ + || fail "shared pool: could not read coordinator instance labels" + orphans="$(printf %s "$workers" | jq -r --argjson known "$known" \ + '[.items[] | select(([.metadata.labels["posthog.com/trino-instance"]] | inside($known)) | not)] | length')" \ + || fail "shared pool: could not compare workers against coordinators" + [ "${orphans:-0}" = "0" ] || fail "shared pool: $orphans worker pod(s) have no coordinator" + + log "shared pool OK [structure]: $ready ready instance(s), each with its own service and workers" + + # Tenant admission, when the cell has the Gateway restriction on. This is the + # user-visible end of the publication barrier: with the gate enabled a + # warehouse is NOT reported ready until its publication has committed, so a + # ready warehouse is evidence that every serving member acknowledged its + # configuration - not merely that a catalog row exists. + [ "${E2E_TRINO_POOL_TENANT_ADMISSION:-0}" = "1" ] || { + log "SKIP shared-pool tenant admission (set E2E_TRINO_POOL_TENANT_ADMISSION=1 on a cell with the gate on)" + return 0 + } + org="${E2E_TRINO_POOL_ORG:?E2E_TRINO_POOL_TENANT_ADMISSION=1 needs E2E_TRINO_POOL_ORG}" + a=0 state="" + while [ "$a" -lt 30 ]; do + state="$(curl -fsS -H "$H" "$API/api/v1/orgs/$org" | jq -r '.trino.state // ""')" || state="" + [ "$state" = "ready" ] && break + sleep 10; a=$((a + 1)) + done + [ "$state" = "ready" ] || fail "shared pool: $org is '$state', want ready once its publication commits" + log "shared pool OK [admission]: $org is admitted (publication committed on every serving member)" + + # And the end the user actually experiences: a statement, run with the + # caller's OWN credentials, returning a result. Structure and admission are + # both upstream of this and neither implies it - a member can be admitted and + # still answer nothing. + # + # The credentials come from the caller (an existing login for that org), so + # this introduces no canary warehouse and no new secret. Without them the + # query is skipped and said to be skipped, rather than quietly reported as + # passing. + [ -n "${E2E_TRINO_POOL_USER:-}" ] && [ -n "${E2E_TRINO_POOL_PASSWORD:-}" ] || { + log "SKIP shared-pool query (set E2E_TRINO_POOL_USER/E2E_TRINO_POOL_PASSWORD to an existing login of $org)" + return 0 + } + endpoint="${E2E_TRINO_POOL_URL:?E2E_TRINO_POOL_USER needs E2E_TRINO_POOL_URL}" + answer="$(pool_scalar "$endpoint" "$E2E_TRINO_POOL_USER" "$E2E_TRINO_POOL_PASSWORD" \ + 'SELECT count(*) FROM (VALUES 1, 2, 3) AS t(x)')" \ + || fail "shared pool: the admitted tenant could not run a statement" + [ "$answer" = "3" ] || fail "shared pool: query returned '$answer', want 3" + log "shared pool OK [query]: $E2E_TRINO_POOL_USER ran a statement and got its result" +} + +# pool_scalar runs one statement through Trino's paging statement protocol and +# prints the first column of the first row. Every page carries the same +# credentials; an error in any page fails the call. +pool_scalar() { # endpoint user password sql + endpoint="$1" user="$2" password="$3" sql="$4" + response="$(curl --connect-timeout 5 --max-time 60 -fsS --user "$user:$password" \ + -H "X-Trino-User: $user" -H 'X-Trino-Time-Zone: UTC' \ + --data-binary "$sql" "$endpoint/v1/statement")" || return 1 + rows='[]' + while :; do + message="$(printf %s "$response" | jq -r '.error.message // empty')" + [ -z "$message" ] || { echo "$message" >&2; return 1; } + rows="$(printf %s "$response" | jq -c --argjson rows "$rows" '$rows + (.data // [])')" + next="$(printf %s "$response" | jq -r '.nextUri // empty')" + [ -n "$next" ] || break + response="$(curl --connect-timeout 5 --max-time 60 -fsS --user "$user:$password" \ + -H "X-Trino-User: $user" "$next")" || return 1 + done + printf %s "$rows" | jq -r '.[0][0] // empty' +} + hot_idle_reporting_and_cap() { # org org="$1" log "hot-idle reporting + cap sweep on $org" @@ -4799,6 +4993,15 @@ engine_main() { # ---- compute-usage billing pull API (meter → buffer → GET → ack) ---- compute_usage_pull_api "$CNPG" "$cnpg_pw" + # ---- shared Trino compute pool ---- + # Inert while disabled; the active acceptance path runs on a pool-enabled + # cluster (E2E_TRINO_POOL=1). + if [ "${E2E_TRINO_POOL:-0}" = "1" ]; then + trino_shared_pool_active + else + trino_shared_pool_disabled + fi + # ---- hot-idle pool reporting + per-org cap sweep ---- hot_idle_reporting_and_cap "$CNPG" diff --git a/tests/mw-dev/e2e/trino.sh b/tests/mw-dev/e2e/trino.sh index ff7772747..94254e413 100644 --- a/tests/mw-dev/e2e/trino.sh +++ b/tests/mw-dev/e2e/trino.sh @@ -91,7 +91,21 @@ trino_query() { # principal password sql rows='[]' while :; do err="$(printf %s "$response" | jq -r '.error.message // empty')" - [ -z "$err" ] || { echo "$err" >&2; return 1; } + if [ -n "$err" ]; then + echo "$err" >&2 + # Show bounded exception classes, not nested messages, SQL, URLs, or credentials. + printf %s "$response" | jq -c ' + def matched($pattern): if type == "string" and test($pattern) then . else null end; + { + queryId: (.id | matched("^[0-9]{8}_[0-9]{6}_[0-9]+_[a-z0-9]+$")), + errorName: (.error.errorName | matched("^[A-Z][A-Z0-9_]{0,127}$")), + errorType: (.error.errorType | matched("^[A-Z][A-Z0-9_]{0,63}$")), + errorCode: (.error.errorCode | if type == "number" then . else null end), + causeTypes: [limit(12; .error.failureInfo | recurse(.cause // empty) | + .type | matched("^[A-Za-z_$][A-Za-z0-9_.$]{0,255}$") | select(. != null))] + }' >&2 2>/dev/null || true + return 1 + fi rows="$(printf %s "$response" | jq -c --argjson rows "$rows" '$rows + (.data // [])')" next="$(printf %s "$response" | jq -r '.nextUri // empty')" [ -n "$next" ] || break diff --git a/tests/mw-dev/trino_query_diagnostics_test.go b/tests/mw-dev/trino_query_diagnostics_test.go new file mode 100644 index 000000000..3314e9ee8 --- /dev/null +++ b/tests/mw-dev/trino_query_diagnostics_test.go @@ -0,0 +1,93 @@ +package e2emwdev_test + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestTrinoQueryFailureDiagnostics(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Fatal("jq is required for the Trino harness fixture") + } + raw, err := os.ReadFile("e2e/trino.sh") + if err != nil { + t.Fatal(err) + } + script := string(raw) + start := strings.Index(script, "trino_query() {") + end := strings.Index(script, "\nscalar() {") + if start < 0 || end <= start { + t.Fatal("Trino statement helper is missing") + } + for _, paged := range []bool{false, true} { + t.Run(map[bool]string{false: "initial", true: "paged"}[paged], func(t *testing.T) { + response := `{"id":"20260101_000000_00001_abcde","infoUri":"https://private.example.test/query","error":{"message":"Failed to finish Hoglake Parquet file","errorName":"HOGLAKE_WRITE_ERROR","errorType":"EXTERNAL","errorCode":123,"failureInfo":{"type":"io.trino.spi.TrinoException","message":"password=fixture-secret","stack":["private.example.test"],"cause":{"type":"java.io.IOException","message":"s3://fixture-private-bucket/key","cause":{"type":"software.amazon.awssdk.services.s3.model.S3Exception","message":"token=fixture-secret","cause":{"type":"invalid type with private.example.test"}}}}}}` + code := `set -eu +CA=fixture-ca +TRINO=https://coordinator.example.test +curl() { + printf 'request\n' >> "$TEST_CALLS" + if [ "$TEST_PAGED" = true ] && [ "$(wc -l < "$TEST_CALLS" | tr -d ' ')" = 1 ]; then + printf '%s\n' '{"nextUri":"https://coordinator.example.test/next"}' + else + printf '%s\n' "$TEST_RESPONSE" + fi +} +` + script[start:end] + "\ntrino_query fixture-user fixture-password 'INSERT INTO fixture VALUES 7'\n" + calls := filepath.Join(t.TempDir(), "calls") + cmd := exec.Command("sh", "-c", code) + cmd.Env = append(os.Environ(), "TEST_RESPONSE="+response, "TEST_CALLS="+calls, "TEST_PAGED="+map[bool]string{false: "false", true: "true"}[paged]) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err == nil { + t.Fatal("failed query incorrectly succeeded") + } + if stdout.Len() != 0 { + t.Fatalf("failed query emitted result rows: %s", stdout.String()) + } + for _, expected := range []string{ + "Failed to finish Hoglake Parquet file", "20260101_000000_00001_abcde", + "HOGLAKE_WRITE_ERROR", "EXTERNAL", "123", "io.trino.spi.TrinoException", + "java.io.IOException", "software.amazon.awssdk.services.s3.model.S3Exception", + } { + if !strings.Contains(stderr.String(), expected) { + t.Errorf("missing diagnostic %q: %s", expected, stderr.String()) + } + } + for _, sensitive := range []string{"private.example.test", "fixture-secret", "fixture-private-bucket", "fixture-password", "INSERT INTO"} { + if strings.Contains(stderr.String(), sensitive) { + t.Errorf("diagnostics exposed %q", sensitive) + } + } + requests, err := os.ReadFile(calls) + if err != nil { + t.Fatal(err) + } + want := 1 + if paged { + want = 2 + } + if got := strings.Count(string(requests), "request\n"); got != want { + t.Fatalf("failed statement was retried: got %d requests, want %d", got, want) + } + }) + } + for _, failureInfo := range []string{"null", `"malformed"`} { + t.Run("incomplete_"+failureInfo, func(t *testing.T) { + cmd := exec.Command("sh", "-c", `set -eu +CA=fixture-ca +TRINO=https://coordinator.example.test +curl() { printf '%s\n' "$TEST_RESPONSE"; } +`+script[start:end]+"\ntrino_query fixture fixture 'SELECT 1'\n") + cmd.Env = append(os.Environ(), `TEST_RESPONSE={"error":{"message":"original error","failureInfo":`+failureInfo+`}}`) + out, err := cmd.CombinedOutput() + if err == nil || !strings.Contains(string(out), "original error") { + t.Fatalf("incomplete diagnostics hid the original failure: %v %s", err, out) + } + }) + } +} diff --git a/tests/trinocatalog/publisher_postgres_test.go b/tests/trinocatalog/publisher_postgres_test.go new file mode 100644 index 000000000..b4626454e --- /dev/null +++ b/tests/trinocatalog/publisher_postgres_test.go @@ -0,0 +1,461 @@ +//go:build linux || darwin + +// Real-PostgreSQL tests for the fenced catalog publisher. The publisher writes +// the catalog store that Trino coordinators read, so its transaction shape, +// fence and journal semantics cannot be established against a fake: they are +// row locks, unique indexes and serialization behavior. +package trinocatalog_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "sync" + "testing" + "time" + + _ "github.com/lib/pq" + "github.com/posthog/duckgres/controlplane/trinocatalog" + integrationtest "github.com/posthog/duckgres/tests/integration" +) + +var ensurePostgresOnce sync.Once + +const testCell = "cell-001" + +func newCatalogStore(t *testing.T) (*sql.DB, string) { + t.Helper() + + dsn := os.Getenv("DUCKGRES_TEST_PG_DSN") + if dsn == "" { + ensurePostgres(t) + dsn = "host=127.0.0.1 port=35432 user=postgres password=postgres dbname=testdb sslmode=disable" + } + admin, err := sql.Open("postgres", dsn) + if err != nil { + t.Fatalf("open postgres: %v", err) + } + t.Cleanup(func() { _ = admin.Close() }) + + schema := fmt.Sprintf("catalog_store_%d", time.Now().UnixNano()) + if _, err := admin.Exec(`CREATE SCHEMA ` + schema); err != nil { + t.Fatalf("create schema: %v", err) + } + t.Cleanup(func() { _, _ = admin.Exec(`DROP SCHEMA IF EXISTS ` + schema + ` CASCADE`) }) + + db, err := sql.Open("postgres", dsn+" search_path="+schema) + if err != nil { + t.Fatalf("open schema connection: %v", err) + } + db.SetMaxOpenConns(8) + t.Cleanup(func() { _ = db.Close() }) + return db, schema +} + +func ensurePostgres(t *testing.T) { + t.Helper() + var err error + ensurePostgresOnce.Do(func() { + if integrationtest.IsPostgresRunning(35432) { + return + } + err = integrationtest.StartPostgresContainer() + }) + if err != nil { + t.Fatalf("start postgres container: %v", err) + } +} + +func newPublisher(t *testing.T, db *sql.DB, identity string, epoch int64) *trinocatalog.Publisher { + t.Helper() + publisher, err := trinocatalog.NewPublisher(db, testCell, identity, epoch) + if err != nil { + t.Fatalf("new publisher: %v", err) + } + return publisher +} + +// unclaimed returns a store whose schema exists but whose cell has no writer +// yet. Epoch 0 with an empty identity is "nobody has ever written here". +func unclaimed(t *testing.T) (*sql.DB, *trinocatalog.Publisher) { + t.Helper() + db, _ := newCatalogStore(t) + publisher := newPublisher(t, db, "duckgres-test", 1) + if err := publisher.EnsureSchema(context.Background()); err != nil { + t.Fatalf("ensure schema: %v", err) + } + return db, publisher +} + +// bootstrapped additionally claims the cell. Claiming is always explicit: a +// publisher never becomes the writer as a side effect of a mutation. +func bootstrapped(t *testing.T) (*sql.DB, *trinocatalog.Publisher) { + t.Helper() + db, publisher := unclaimed(t) + if _, err := publisher.Takeover(context.Background()); err != nil { + t.Fatalf("takeover: %v", err) + } + return db, publisher +} + +// A publisher that has not claimed the cell must not be able to write, even +// when nobody else holds it. +func TestUnclaimedCellRejectsMutations(t *testing.T) { + _, publisher := unclaimed(t) + if _, err := publisher.Apply(context.Background(), addCatalog("org_a")); !errors.Is(err, trinocatalog.ErrNotWriter) { + t.Fatalf("unclaimed apply error = %v, want ErrNotWriter", err) + } +} + +func addCatalog(name string) trinocatalog.Mutation { + return trinocatalog.Mutation{ + OperationID: "op-" + name, + Operation: trinocatalog.OperationAddOrReplace, + CatalogName: name, + ConnectorName: "ducklake", + Properties: map[string]string{ + "ducklake.metadata.connection-url": "jdbc:postgresql://db:5432/lake", + "ducklake.data-path": "s3://bucket/prefix/", + }, + } +} + +func readState(t *testing.T, db *sql.DB) (revision int64, epoch int64, identity string, count int) { + t.Helper() + row := db.QueryRow(`SELECT revision, writer_epoch, writer_identity, catalog_count FROM trino_catalog_writer_state WHERE cell_id = $1`, testCell) + if err := row.Scan(&revision, &epoch, &identity, &count); err != nil { + t.Fatalf("read writer state: %v", err) + } + return revision, epoch, identity, count +} + +// The schema is created by the publisher because a managed-reader coordinator +// runs no DDL at all. +func TestEnsureSchemaIsIdempotent(t *testing.T) { + db, publisher := bootstrapped(t) + if err := publisher.EnsureSchema(context.Background()); err != nil { + t.Fatalf("second ensure schema: %v", err) + } + for _, table := range []string{"trino_catalogs", "trino_catalog_writer_state", "trino_catalog_journal"} { + var exists bool + if err := db.QueryRow(`SELECT to_regclass($1) IS NOT NULL`, table).Scan(&exists); err != nil || !exists { + t.Fatalf("table %s missing (err=%v)", table, err) + } + } +} + +// Seeding a writer-state row at catalog_count 0 against a store that already +// holds catalogs makes every reader's completeness check fail, which freezes the +// whole fleet on last-good state. The seed must count the existing rows. +func TestSeedsWriterStateFromExistingCatalogRows(t *testing.T) { + db, publisher := unclaimed(t) + for _, name := range []string{"org_a", "org_b", "org_c"} { + if _, err := db.Exec(`INSERT INTO trino_catalogs (cell_id, catalog_name, connector_name, catalog_version, properties) VALUES ($1,$2,'ducklake','v','{}')`, testCell, name); err != nil { + t.Fatalf("seed catalog row: %v", err) + } + } + // A row for a different cell must not be counted. + if _, err := db.Exec(`INSERT INTO trino_catalogs (cell_id, catalog_name, connector_name, catalog_version, properties) VALUES ('other-cell','org_z','ducklake','v','{}')`); err != nil { + t.Fatalf("seed foreign catalog row: %v", err) + } + + if _, err := publisher.State(context.Background()); err != nil { + t.Fatalf("state: %v", err) + } + revision, _, _, count := readState(t, db) + if revision != 0 { + t.Fatalf("seeded revision = %d, want 0", revision) + } + if count != 3 { + t.Fatalf("seeded catalog_count = %d, want 3", count) + } +} + +func TestApplyAdvancesRevisionAndRecomputesCount(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + + first, err := publisher.Apply(ctx, addCatalog("org_a")) + if err != nil { + t.Fatalf("apply: %v", err) + } + if first.Revision != 1 || first.Replayed { + t.Fatalf("first apply = %+v, want revision 1", first) + } + second, err := publisher.Apply(ctx, addCatalog("org_b")) + if err != nil { + t.Fatalf("apply: %v", err) + } + if second.Revision != 2 { + t.Fatalf("second apply revision = %d, want 2", second.Revision) + } + + revision, epoch, identity, count := readState(t, db) + if revision != 2 || epoch != 1 || identity != "duckgres-test" || count != 2 { + t.Fatalf("writer state = (%d,%d,%q,%d)", revision, epoch, identity, count) + } + + var version string + if err := db.QueryRow(`SELECT catalog_version FROM trino_catalogs WHERE cell_id=$1 AND catalog_name='org_a'`, testCell).Scan(&version); err != nil { + t.Fatalf("read catalog: %v", err) + } + // The version has to be the Trino content hash, not an invented value. + if want := trinocatalog.Mutation(addCatalog("org_a")).CatalogVersion(); version != want { + t.Fatalf("catalog_version = %q, want %q", version, want) + } +} + +func TestRemoveDeletesTheRowAndJournalsANullVersion(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + removal := trinocatalog.Mutation{OperationID: "op-remove", Operation: trinocatalog.OperationRemove, CatalogName: "org_a"} + result, err := publisher.Apply(ctx, removal) + if err != nil { + t.Fatalf("remove: %v", err) + } + if result.Revision != 2 || result.CatalogCount != 0 { + t.Fatalf("remove result = %+v", result) + } + var version sql.NullString + if err := db.QueryRow(`SELECT catalog_version FROM trino_catalog_journal WHERE cell_id=$1 AND operation_id='op-remove'`, testCell).Scan(&version); err != nil { + t.Fatalf("read journal: %v", err) + } + if version.Valid { + t.Fatalf("REMOVE journalled a catalog_version %q", version.String) + } +} + +// A lost COMMIT response must be resolved from the journal, never by applying +// the mutation a second time or compensating with a DROP. +func TestReplayReturnsTheRecordedRevision(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + first, err := publisher.Apply(ctx, addCatalog("org_a")) + if err != nil { + t.Fatalf("apply: %v", err) + } + replay, err := publisher.Apply(ctx, addCatalog("org_a")) + if err != nil { + t.Fatalf("replay: %v", err) + } + if !replay.Replayed || replay.Revision != first.Revision { + t.Fatalf("replay = %+v, want the recorded revision %d", replay, first.Revision) + } + revision, _, _, _ := readState(t, db) + if revision != first.Revision { + t.Fatalf("replay advanced the revision to %d", revision) + } + + resolved, err := publisher.ResolveOperation(ctx, "op-org_a") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if resolved == nil || resolved.Revision != first.Revision { + t.Fatalf("resolve = %+v, want revision %d", resolved, first.Revision) + } + missing, err := publisher.ResolveOperation(ctx, "op-never-happened") + if err != nil { + t.Fatalf("resolve missing: %v", err) + } + if missing != nil { + t.Fatalf("resolve of an unknown operation returned %+v", missing) + } +} + +// Same operation id with different content is a bug in the caller, not a +// replay: silently applying it would publish an unintended definition. +func TestChangedIntentUnderTheSameOperationIDIsAConflict(t *testing.T) { + _, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + changed := addCatalog("org_a") + changed.Properties["ducklake.data-path"] = "s3://other-bucket/prefix/" + if _, err := publisher.Apply(ctx, changed); !errors.Is(err, trinocatalog.ErrIntentChanged) { + t.Fatalf("changed intent error = %v, want ErrIntentChanged", err) + } +} + +func TestFencedWriterCannotWrite(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + + successor := newPublisher(t, db, "duckgres-successor", 2) + if _, err := successor.Takeover(ctx); err != nil { + t.Fatalf("takeover: %v", err) + } + + // The old publisher is now fenced and must not write anything at all. + if _, err := publisher.Apply(ctx, addCatalog("org_b")); !errors.Is(err, trinocatalog.ErrFenced) { + t.Fatalf("stale writer error = %v, want ErrFenced", err) + } + revision, epoch, identity, count := readState(t, db) + if revision != 1 || epoch != 2 || identity != "duckgres-successor" || count != 1 { + t.Fatalf("stale writer changed state: (%d,%d,%q,%d)", revision, epoch, identity, count) + } + if _, err := successor.Apply(ctx, addCatalog("org_b")); err != nil { + t.Fatalf("successor apply: %v", err) + } +} + +// Root integration decision 2: a mutation verifies the exact epoch AND identity. +// It never claims a higher epoch as a side effect, because that would let a +// process that merely believes it is newer seize the cell mid-write. +func TestMutationNeverClaimsAHigherEpochImplicitly(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + ambitious := newPublisher(t, db, "duckgres-ambitious", 5) + if _, err := ambitious.Apply(ctx, addCatalog("org_b")); !errors.Is(err, trinocatalog.ErrNotWriter) { + t.Fatalf("implicit takeover error = %v, want ErrNotWriter", err) + } + _, epoch, identity, _ := readState(t, db) + if epoch != 1 || identity != "duckgres-test" { + t.Fatalf("an unapproved mutation moved the fence to (%d,%q)", epoch, identity) + } +} + +// Same epoch, different process: exactly one writer identity owns a cell. +func TestSameEpochDifferentIdentityIsRejected(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + impostor := newPublisher(t, db, "duckgres-impostor", 1) + if _, err := impostor.Apply(ctx, addCatalog("org_b")); !errors.Is(err, trinocatalog.ErrNotWriter) { + t.Fatalf("same-epoch impostor error = %v, want ErrNotWriter", err) + } +} + +func TestTakeoverIsMonotonic(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + if _, err := newPublisher(t, db, "duckgres-successor", 2).Takeover(ctx); err != nil { + t.Fatalf("takeover: %v", err) + } + // An older epoch can never take the cell back. + if _, err := newPublisher(t, db, "duckgres-test", 1).Takeover(ctx); !errors.Is(err, trinocatalog.ErrFenced) { + t.Fatalf("regressing takeover error = %v, want ErrFenced", err) + } + _, epoch, identity, _ := readState(t, db) + if epoch != 2 || identity != "duckgres-successor" { + t.Fatalf("writer fence regressed to (%d,%q)", epoch, identity) + } +} + +// The writer-state row lock is what serializes concurrent publishers. Revisions +// must come out contiguous with no gaps and no duplicates, because the reader +// polls that single number to decide whether to fetch a snapshot. +func TestConcurrentPublishersProduceContiguousRevisions(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.State(ctx); err != nil { + t.Fatalf("state: %v", err) + } + + const concurrency = 8 + revisions := make([]int64, concurrency) + errs := make([]error, concurrency) + var wg sync.WaitGroup + start := make(chan struct{}) + for index := 0; index < concurrency; index++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + <-start + result, err := publisher.Apply(ctx, addCatalog(fmt.Sprintf("org_%d", index))) + revisions[index], errs[index] = result.Revision, err + }(index) + } + close(start) + wg.Wait() + + seen := map[int64]bool{} + for index, err := range errs { + if err != nil { + t.Fatalf("concurrent apply %d: %v", index, err) + } + if seen[revisions[index]] { + t.Fatalf("revision %d was handed out twice", revisions[index]) + } + seen[revisions[index]] = true + } + for expected := int64(1); expected <= concurrency; expected++ { + if !seen[expected] { + t.Fatalf("revision %d is missing; revisions are not contiguous", expected) + } + } + revision, _, _, count := readState(t, db) + if revision != concurrency || count != concurrency { + t.Fatalf("final state revision=%d count=%d, want %d/%d", revision, count, concurrency, concurrency) + } +} + +// The reader's completeness check compares catalog_count against the rows it +// read, so the count has to be recomputed inside the mutation transaction +// rather than incremented optimistically. +func TestCatalogCountIsRecomputedInsideTheTransaction(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + // Something outside the publisher (a legacy writer during the migration + // bridge) adds a row. The next mutation must reconcile the count, not + // carry the stale one forward. + if _, err := db.Exec(`INSERT INTO trino_catalogs (cell_id, catalog_name, connector_name, catalog_version, properties) VALUES ($1,'org_legacy','ducklake','v','{}')`, testCell); err != nil { + t.Fatalf("legacy insert: %v", err) + } + if _, err := publisher.Apply(ctx, addCatalog("org_b")); err != nil { + t.Fatalf("apply: %v", err) + } + _, _, _, count := readState(t, db) + if count != 3 { + t.Fatalf("catalog_count = %d, want 3", count) + } +} + +// Two publishers at the SAME epoch with different identities are not a +// takeover, they are a collision. Allowing the second to overwrite the identity +// let both pass the mutation fence afterwards - the precise ambiguity the +// identity half of the fence exists to remove. Only a strictly higher epoch may +// claim the cell. +func TestTakeoverRefusesAnEqualEpochHeldByAnotherWriter(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + if _, err := publisher.Apply(ctx, addCatalog("org_a")); err != nil { + t.Fatalf("apply: %v", err) + } + + impostor := newPublisher(t, db, "duckgres-impostor", 1) + if _, err := impostor.Takeover(ctx); !errors.Is(err, trinocatalog.ErrFenced) { + t.Fatalf("equal-epoch takeover error = %v, want ErrFenced", err) + } + // The original writer still owns the cell and can still write. + _, epoch, identity, _ := readState(t, db) + if epoch != 1 || identity != "duckgres-test" { + t.Fatalf("writer fence moved to (%d,%q)", epoch, identity) + } + if _, err := publisher.Apply(ctx, addCatalog("org_b")); err != nil { + t.Fatalf("original writer lost its cell: %v", err) + } + // And the impostor is still refused on the mutation path. + if _, err := impostor.Apply(ctx, addCatalog("org_c")); !errors.Is(err, trinocatalog.ErrNotWriter) { + t.Fatalf("impostor apply error = %v, want ErrNotWriter", err) + } +} diff --git a/tests/trinocatalog/writer_bridge_postgres_test.go b/tests/trinocatalog/writer_bridge_postgres_test.go new file mode 100644 index 000000000..58e16bb79 --- /dev/null +++ b/tests/trinocatalog/writer_bridge_postgres_test.go @@ -0,0 +1,211 @@ +//go:build linux || darwin + +package trinocatalog_test + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/posthog/duckgres/controlplane/trinocatalog" + "github.com/posthog/duckgres/controlplane/trinopool" +) + +// The provisioner publishes a catalog as a property map that includes +// connector.name. The store keeps the connector in its own column and hashes +// only the remaining properties, exactly as the coordinator does - so the +// version it writes has to equal the version the coordinator computes for the +// same catalog. +func TestPublishedCatalogVersionMatchesTheCoordinatorsOwn(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + + properties := map[string]string{ + "ducklake.metadata.connection-url": "jdbc:postgresql://db:5432/lake", + "ducklake.data-path": "s3://bucket/prefix/", + } + if _, err := publisher.Apply(ctx, trinocatalog.Mutation{ + OperationID: "catalog.org_acme.first", Operation: trinocatalog.OperationAddOrReplace, + CatalogName: "org_acme", ConnectorName: "ducklake", Properties: properties, + }); err != nil { + t.Fatalf("publish: %v", err) + } + + var version, connector string + if err := db.QueryRow( + `SELECT catalog_version, connector_name FROM trino_catalogs WHERE cell_id=$1 AND catalog_name='org_acme'`, + testCell).Scan(&version, &connector); err != nil { + t.Fatalf("read catalog: %v", err) + } + if connector != "ducklake" { + t.Fatalf("connector = %q", connector) + } + if want := trinopool.CatalogVersion("org_acme", "ducklake", properties); version != want { + t.Fatalf("catalog_version = %q, want the coordinator's own hash %q", version, want) + } +} + +// Republishing an unchanged catalog must not advance the revision: coordinators +// poll that number, and a revision that moves on every reconcile tick would +// make every coordinator refetch the whole snapshot forever. +func TestRepublishingAnUnchangedCatalogIsAReplay(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + + mutation := trinocatalog.Mutation{ + Operation: trinocatalog.OperationAddOrReplace, CatalogName: "org_acme", + ConnectorName: "ducklake", + Properties: map[string]string{"ducklake.data-path": "s3://bucket/prefix/"}, + } + // The bridge derives the operation id from the intent, which is what makes + // an unchanged republication a replay. + mutation.OperationID = "catalog.org_acme." + mutation.PayloadHash()[:32] + + first, err := publisher.Apply(ctx, mutation) + if err != nil { + t.Fatalf("publish: %v", err) + } + second, err := publisher.Apply(ctx, mutation) + if err != nil { + t.Fatalf("republish: %v", err) + } + if !second.Replayed || second.Revision != first.Revision { + t.Fatalf("republication advanced the revision: %+v -> %+v", first, second) + } + + // A CHANGED intent is a different operation and does advance it. + changed := mutation + changed.Properties = map[string]string{"ducklake.data-path": "s3://other/prefix/"} + changed.OperationID = "catalog.org_acme." + changed.PayloadHash()[:32] + third, err := publisher.Apply(ctx, changed) + if err != nil { + t.Fatalf("publish changed: %v", err) + } + if third.Revision <= first.Revision { + t.Fatalf("a changed catalog did not advance the revision: %+v", third) + } + + var revision int64 + if err := db.QueryRow(`SELECT revision FROM trino_catalog_writer_state WHERE cell_id=$1`, testCell).Scan(&revision); err != nil { + t.Fatalf("read revision: %v", err) + } + if revision != third.Revision { + t.Fatalf("writer state revision = %d, want %d", revision, third.Revision) + } +} + +// The reader's completeness check compares catalog_count with the rows it read, +// so a drop has to leave both consistent within one transaction. +func TestDropLeavesTheStoreConsistent(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + + for _, name := range []string{"org_a", "org_b"} { + if _, err := publisher.Apply(ctx, addCatalog(name)); err != nil { + t.Fatalf("publish %s: %v", name, err) + } + } + if _, err := publisher.Apply(ctx, trinocatalog.Mutation{ + OperationID: "catalog.org_a.drop", Operation: trinocatalog.OperationRemove, CatalogName: "org_a", + }); err != nil { + t.Fatalf("drop: %v", err) + } + + var count int + var stored int + if err := db.QueryRow(`SELECT count(*) FROM trino_catalogs WHERE cell_id=$1`, testCell).Scan(&count); err != nil { + t.Fatalf("count rows: %v", err) + } + if err := db.QueryRow(`SELECT catalog_count FROM trino_catalog_writer_state WHERE cell_id=$1`, testCell).Scan(&stored); err != nil { + t.Fatalf("read catalog_count: %v", err) + } + if count != 1 || stored != count { + t.Fatalf("rows=%d catalog_count=%d, want both 1", count, stored) + } +} + +// A reader that polls the revision must see a consistent (revision, rows, +// count) triple. This is the read the coordinator performs. +func TestReaderSnapshotIsComplete(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + for _, name := range []string{"org_a", "org_b", "org_c"} { + if _, err := publisher.Apply(ctx, addCatalog(name)); err != nil { + t.Fatalf("publish %s: %v", name, err) + } + } + + tx, err := db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) + if err != nil { + t.Fatalf("begin snapshot: %v", err) + } + defer func() { _ = tx.Rollback() }() + + var revision int64 + var expected int + if err := tx.QueryRow(`SELECT revision, catalog_count FROM trino_catalog_writer_state WHERE cell_id=$1`, testCell). + Scan(&revision, &expected); err != nil { + t.Fatalf("read writer state: %v", err) + } + var actual int + if err := tx.QueryRow(`SELECT count(*) FROM trino_catalogs WHERE cell_id=$1`, testCell).Scan(&actual); err != nil { + t.Fatalf("count catalogs: %v", err) + } + if actual != expected { + t.Fatalf("snapshot is incomplete: %d rows, catalog_count %d", actual, expected) + } + if revision != 3 { + t.Fatalf("revision = %d, want 3", revision) + } +} + +// Create, drop, then create again with IDENTICAL properties. With a +// content-only operation id the third call hit the journal and returned the +// first create's revision without writing anything: the catalog was dropped and +// never republished, so no coordinator ever saw it again. +func TestRecreateAfterDropIsRepublished(t *testing.T) { + db, publisher := bootstrapped(t) + ctx := context.Background() + + create := func() trinocatalog.Mutation { + return trinocatalog.Mutation{ + Operation: trinocatalog.OperationAddOrReplace, CatalogName: "org_acme", + ConnectorName: "ducklake", + Properties: map[string]string{"ducklake.data-path": "s3://bucket/prefix/"}, + } + } + // The bridge's identity scheme: the store's current revision plus the + // intent, so an intent repeated after other commits is a NEW operation. + apply := func(mutation trinocatalog.Mutation) trinocatalog.Result { + t.Helper() + state, err := publisher.State(ctx) + if err != nil { + t.Fatalf("state: %v", err) + } + mutation.OperationID = fmt.Sprintf("catalog.%s.r%d.%s", mutation.CatalogName, state.Revision, mutation.PayloadHash()[:16]) + result, err := publisher.Apply(ctx, mutation) + if err != nil { + t.Fatalf("apply: %v", err) + } + return result + } + + first := apply(create()) + apply(trinocatalog.Mutation{Operation: trinocatalog.OperationRemove, CatalogName: "org_acme"}) + third := apply(create()) + + if third.Replayed { + t.Fatal("the recreate was treated as a replay of the original create") + } + if third.Revision <= first.Revision { + t.Fatalf("recreate revision %d did not advance past %d", third.Revision, first.Revision) + } + var present bool + if err := db.QueryRow(`SELECT count(*) = 1 FROM trino_catalogs WHERE cell_id=$1 AND catalog_name='org_acme'`, testCell).Scan(&present); err != nil { + t.Fatalf("read catalog: %v", err) + } + if !present { + t.Fatal("the recreated catalog is not in the store") + } +} diff --git a/tools/gatewaywire/PoolWireFixtures.java b/tools/gatewaywire/PoolWireFixtures.java new file mode 100644 index 000000000..75981b189 --- /dev/null +++ b/tools/gatewaywire/PoolWireFixtures.java @@ -0,0 +1,89 @@ +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import io.trino.gateway.ha.transaction.PoolStore; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * Serializes the Gateway's REAL pooled-lifecycle records with Jackson and writes the resulting + * wire JSON to disk. The duckgres Go client consumes these files as testdata, so its decoding is + * pinned to what the Java side actually produces rather than to a hand-written copy of a contract + * document. Every identifier here is synthetic. + */ +public final class PoolWireFixtures +{ + private PoolWireFixtures() {} + + public static void main(String[] args) + throws Exception + { + Path out = Path.of(args[0]); + Files.createDirectories(out); + ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); + + UUID incarnation = UUID.fromString("11111111-1111-4111-8111-111111111111"); + + PoolStore.PoolState pool = new PoolStore.PoolState( + 1, "pool-001", "POOLED", 7L, 19L, + 3, 3, 1, 1, + "r-42", "r-41", true, + Map.of("PREPARING", 1L, "ACTIVE", 3L, "RETIRED", 4L, "LOST", 1L), + 3L, 4L, 1L, 0L, 0L, List.of(), false); + write(mapper, out.resolve("pool_state.json"), pool); + + PoolStore.Member member = new PoolStore.Member( + 1, "pool-001", "i-0007", incarnation, "pool-001-i-0007", + "https://i-0007.pool.invalid:8443", "https://i-0007.external.invalid:8443", + "ACTIVE", 4L, 7L, + "pod-uid-0007", "boot-0007", "node-0007", "abcde", + "r-42", "r-42", "auth-9", false, null, null, + 0L, 0L, 0L, false, false, true, 19L, false); + write(mapper, out.resolve("member.json"), member); + write(mapper, out.resolve("members.json"), List.of(member)); + + PoolStore.Obligations obligations = new PoolStore.Obligations( + 1, "i-0007", incarnation, "DRAINING", 5L, 2L, 1L, 3L, false, false); + write(mapper, out.resolve("obligations.json"), obligations); + + PoolStore.Publication publication = new PoolStore.Publication( + 1, "pub-1", "pool-001", "tenant-a", "r-42", 19L, "OPEN", + List.of("i-0007"), + List.of(new PoolStore.PublicationReceipt("i-0007", incarnation, "pod-uid-0007", "boot-0007", "r-42", "fingerprint-1")), + List.of(), "PENDING", null, false); + write(mapper, out.resolve("publication.json"), publication); + + // The read path echoes the published logins; a write result carries the + // count and hash instead, so a recorded idempotent step stays bounded + // however many logins a tenant has. + PoolStore.TenantAdmission tenant = new PoolStore.TenantAdmission( + 1, "pool-001", "tenant-a", "ADMITTED", "r-42", "pub-1", + "binding-1", 2, "0".repeat(64), + List.of("warehouse-one", "warehouse-one.alice"), false); + write(mapper, out.resolve("tenant_admission.json"), tenant); + + PoolStore.FailureReceipt failure = new PoolStore.FailureReceipt( + 1, "pool-001", "i-0007", incarnation, "PROCESS_TERMINATED", + mapper.readTree("{\"source\":\"kubernetes-pod-absent\"}"), + 0L, 1L, 2L, "2026-09-18T00:00:00Z"); + write(mapper, out.resolve("failure_receipt.json"), failure); + + PoolStore.OperationHistory history = new PoolStore.OperationHistory( + 1, "op-1", + List.of(new PoolStore.OperationStep( + "admit", "0".repeat(64), 7L, "OK", "2026-09-18T00:00:00Z", + mapper.readTree("{\"phase\":\"ACTIVE\"}")))); + write(mapper, out.resolve("operation_history.json"), history); + + System.out.println("wrote fixtures to " + out.toAbsolutePath()); + } + + private static void write(ObjectMapper mapper, Path path, Object value) + throws Exception + { + Files.writeString(path, mapper.writeValueAsString(value) + "\n"); + } +} diff --git a/tools/gatewaywire/README.md b/tools/gatewaywire/README.md new file mode 100644 index 000000000..34ebfcdf6 --- /dev/null +++ b/tools/gatewaywire/README.md @@ -0,0 +1,33 @@ +# Gateway wire fixtures + +`controlplane/trinogateway/testdata/*.json` is not hand-written. Each file is +produced by serializing the Gateway's real `PoolStore` records with Jackson, so +the Go client's decoding is pinned to what the Java side actually emits rather +than to a copy of a design document. A hand-written fixture would only prove +that the test and the code under test agree with each other. + +Regenerate after any change to the Java records: + +```sh +GATEWAY= +JAVA_HOME= +M2="$HOME/.m2/repository/com/fasterxml/jackson/core" + +CP="$M2/jackson-databind/2.21.5/jackson-databind-2.21.5.jar:\ +$M2/jackson-core/2.21.4/jackson-core-2.21.4.jar:\ +$M2/jackson-annotations/2.21/jackson-annotations-2.21.jar:\ +$GATEWAY/gateway-ha/target/classes" + +"$JAVA_HOME/bin/javac" -cp "$CP" -d /tmp/gatewaywire tools/gatewaywire/PoolWireFixtures.java +"$JAVA_HOME/bin/java" -cp "/tmp/gatewaywire:$CP" PoolWireFixtures \ + controlplane/trinogateway/testdata +``` + +The Gateway checkout must be built first (`./mvnw -pl gateway-ha compile`) so +`target/classes` exists. Every identifier in the generated fixtures is +synthetic. + +The decoder tests use `DisallowUnknownFields`, so a field added on the Java +side fails the Go build-out rather than being silently ignored — which is the +point: a consumer that quietly drops a new field keeps making decisions on a +stale view of the member.