From ed31a9babecb43945543a51496683de4458a7e74 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:05:26 +0200 Subject: [PATCH 1/2] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/2222 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events Antalya-26.6: Datalake catalog auth token profile events # Conflicts: # src/Databases/DataLake/RestCatalog.cpp # src/Databases/DataLake/RestCatalog.h # src/Databases/DataLake/S3TablesCatalog.cpp # src/Databases/DataLake/S3TablesCatalog.h # tests/integration/test_database_iceberg_lakekeeper_catalog/test.py --- src/Common/ProfileEvents.cpp | 1 + src/Databases/DataLake/RestCatalog.cpp | 68 +++++++++++++++++++ src/Databases/DataLake/RestCatalog.h | 16 +++++ src/Databases/DataLake/S3TablesCatalog.cpp | 10 +++ src/Databases/DataLake/S3TablesCatalog.h | 10 +++ .../test.py | 68 +++++++++++++++++++ 6 files changed, 173 insertions(+) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 7e7d268a88ad..0abbaf1dcbb5 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1816,6 +1816,7 @@ The server successfully detected this situation and will download merged part fr \ + #ifdef APPLY_FOR_EXTERNAL_EVENTS #define APPLY_FOR_EVENTS(M) APPLY_FOR_BUILTIN_EVENTS(M) APPLY_FOR_EXTERNAL_EVENTS(M) #else diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index ccd48047d39e..c4a0b4defa18 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -334,6 +334,13 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( const CatalogState & catalog_state, bool update_token, +<<<<<<< HEAD +======= + const String & /*method*/, + const Poco::URI & /*url*/, + const DB::HTTPHeaderEntries & /*extra_headers*/, + const String & /*body*/, +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) bool * used_cached_oauth_token) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, @@ -582,6 +589,7 @@ AccessToken OneLakeCatalog::getValidAccessToken(const CatalogState & catalog_sta std::pair OneLakeCatalog::getCurrentAccessToken() const { +<<<<<<< HEAD const auto state_snapshot = state.get(); const auto token = getValidAccessToken(*state_snapshot, /* force_update */ false); /// A token without a known expiration is reported as expiring shortly, so that the @@ -794,6 +802,8 @@ namespace AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const { +======= +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); @@ -1055,6 +1065,13 @@ BigLakeCatalog::BigLakeCatalog( DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( const CatalogState & catalog_state, bool update_token, +<<<<<<< HEAD +======= + const String & method, + const Poco::URI & url, + const DB::HTTPHeaderEntries & extra_headers, + const String & body, +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) bool * used_cached_oauth_token) const { /// Google Cloud OAuth2 for BigLake. @@ -1094,7 +1111,11 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } +<<<<<<< HEAD return RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); +======= + return RestCatalog::getAuthHeaders(update_token, method, url, extra_headers, body, used_cached_oauth_token); +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1118,6 +1139,25 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); +<<<<<<< HEAD +======= + + if (!google_adc_client_id.empty() && !google_adc_client_secret.empty() && !google_adc_refresh_token.empty()) + { + try + { + return retrieveGoogleCloudAccessTokenFromRefreshToken(); + } + catch (const DB::Exception & e) + { + LOG_DEBUG(log, "Failed to use ADC credentials, falling back to metadata service: {}", e.what()); + } + } + + /// Fallback to GCP metadata service (works inside GCP infrastructure) + /// https://cloud.google.com/compute/docs/metadata/overview + static constexpr auto DEFAULT_REQUEST_TOKEN_PATH = "/computeMetadata/v1/instance/service-accounts"; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) const auto & context = getContext(); @@ -1252,7 +1292,11 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { +<<<<<<< HEAD auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); +======= + auto result_headers = getAuthHeaders(update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}, &used_cached_oauth_token); +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1917,11 +1961,35 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & .create(credentials); }; +<<<<<<< HEAD + try + { + bool used_cached_oauth_token = false; + auto wb = create_buffer(false, used_cached_oauth_token); + +======= + auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) + { + DB::HTTPHeaderEntries headers = getAuthHeaders(update_token, method, url, extra_headers, body_str, &used_cached_oauth_token); + headers.emplace_back("Content-Type", "application/json"); + return DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withMethod(method) + .withSettings(context->getReadSettings()) + .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) + .withHostFilter(&context->getRemoteHostFilter()) + .withHeaders(headers) + .withOutCallback(out_stream_callback) + .withSkipNotFound(false) + .create(credentials); + }; + try { bool used_cached_oauth_token = false; auto wb = create_buffer(false, used_cached_oauth_token); +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) String response_str; if (!ignore_result) readJSONObjectPossiblyInvalid(response_str, *wb); diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index 077f186395ca..b3f76689c182 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -235,7 +235,15 @@ class RestCatalog : public ICatalog, public DB::WithContext virtual DB::HTTPHeaderEntries getAuthHeaders( const CatalogState & catalog_state, bool update_token, +<<<<<<< HEAD bool * used_cached_oauth_token) const; +======= + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}, + bool * used_cached_oauth_token = nullptr) const; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -362,7 +370,15 @@ class BigLakeCatalog : public RestCatalog DB::HTTPHeaderEntries getAuthHeaders( const CatalogState & catalog_state, bool update_token, +<<<<<<< HEAD bool * used_cached_oauth_token) const override; +======= + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}, + bool * used_cached_oauth_token = nullptr) const override; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 7b1d5ff57a3a..52bb50da5a5c 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -262,7 +262,17 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } +<<<<<<< HEAD namespace +======= +DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( + bool /*update_token*/, + const String & method, + const Poco::URI & url, + const DB::HTTPHeaderEntries & extra_headers, + const String & body, + bool * /*used_cached_oauth_token*/) const +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { /// `signRequestWithAWSV4` returns the full set of headers that the AWS SDK kept on diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index 2bb706df9796..fb7fb520bacd 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -49,6 +49,7 @@ class S3TablesCatalog final : public RestCatalog void dropTable(const String & namespace_name, const String & table_name, bool delete_data) const override; protected: +<<<<<<< HEAD /// Override the network primitives instead of `getAuthHeaders` so the SigV4 signer has /// access to the final URL, method, and request body for canonicalisation. /// `catalog_state` and `auth_headers` are unused here: authentication is derived from the @@ -66,6 +67,15 @@ class S3TablesCatalog final : public RestCatalog Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const override; +======= + DB::HTTPHeaderEntries getAuthHeaders( + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}, + bool * used_cached_oauth_token = nullptr) const override; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) private: const String region; diff --git a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py index d8ac831a78d8..6f05d70ca9c1 100644 --- a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py +++ b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py @@ -490,6 +490,22 @@ def get_auth_token_profile_events(node, query_id): return refreshed, cache_hits +<<<<<<< HEAD +======= +def get_auth_token_profile_events(node, query_id): + node.query("SYSTEM FLUSH LOGS") + refreshed = int(node.query( + f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenRefreshed'] " + f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + )) + cache_hits = int(node.query( + f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenCacheHits'] " + f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + )) + return refreshed, cache_hits + + +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) def test_auth_token_profile_events(started_cluster): node = started_cluster.instances["node1"] @@ -499,6 +515,58 @@ def test_auth_token_profile_events(started_cluster): table_name = f"{test_ref}_table" catalog = load_catalog_impl(started_cluster) +<<<<<<< HEAD +======= + if namespace not in catalog.list_namespaces(): + catalog.create_namespace(namespace) + + schema = Schema( + NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), + NestedField(field_id=2, name="data", field_type=StringType(), required=False), + ) + catalog.create_table( + namespace + (table_name,), + schema=schema, + properties={"write.metadata.compression-codec": "none"}, + ) + + # The catalog client is initialized lazily on the first database access, + # not during CREATE DATABASE. OAuth credentials must use client_id:client_secret + # format; oauth_server_uri points to a mock token endpoint in docker compose. + create_clickhouse_iceberg_database( + started_cluster, + node, + db_name, + additional_settings={ + "catalog_credential": "test:secret", + "oauth_server_uri": MOCK_OAUTH_URL, + }, + ) + + qid1 = f"{test_ref}-show-1-{uuid.uuid4()}" + node.query(f"SHOW TABLES FROM {db_name}", query_id=qid1) + assert table_name in node.query(f"SHOW TABLES FROM {db_name}") + refreshed, cache_hits = get_auth_token_profile_events(node, qid1) + assert refreshed >= 1 + + qid2 = f"{test_ref}-show-2-{uuid.uuid4()}" + node.query(f"SHOW TABLES FROM {db_name}", query_id=qid2) + refreshed, cache_hits = get_auth_token_profile_events(node, qid2) + assert refreshed == 0 and cache_hits >= 1 + + +def test_vended_credentials_cache(started_cluster): + node = started_cluster.instances["node1"] + catalog = load_catalog_impl(started_cluster) + + test_ref = f"test_vended_credentials_cache_{uuid.uuid4().hex[:8]}" + namespace = (f"{test_ref}_namespace",) + table_name = f"{test_ref}_table" + db_name = f"{test_ref}_database" + + +def create_int_table(catalog, namespace, table_name, rows=1): +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) if namespace not in catalog.list_namespaces(): catalog.create_namespace(namespace) From 23b33785529621a84a51ff9f72bdd60b49b333dc Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:59:37 +0200 Subject: [PATCH 2/2] Resolve conflicts in cherry-pick of #2222 The source PR's changes (profile events `DataLakeRestCatalogAuthTokenCachedValid`, `DataLakeRestCatalogAuthTokenRetrieve`, `DataLakeRestCatalogAuthTokenRefreshedMicroseconds`, `DataLakeRestCatalogUnauthorized`, the `used_cached_oauth_token` plumbing in `getAuthHeaders`, the retry-on-401/403 in `sendRequest`, and the `test_auth_token_profile_events` integration test) are already present on antalya-26.8, adapted to the `CatalogState`-based `getAuthHeaders` signature. Keep the antalya-26.8 version of all conflicted files. --- src/Databases/DataLake/RestCatalog.cpp | 68 ------------------- src/Databases/DataLake/RestCatalog.h | 16 ----- src/Databases/DataLake/S3TablesCatalog.cpp | 10 --- src/Databases/DataLake/S3TablesCatalog.h | 10 --- .../test.py | 68 ------------------- 5 files changed, 172 deletions(-) diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index c4a0b4defa18..ccd48047d39e 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -334,13 +334,6 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( const CatalogState & catalog_state, bool update_token, -<<<<<<< HEAD -======= - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/, ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) bool * used_cached_oauth_token) const { fiu_do_on(DB::FailPoints::check_database_datalake_negative, @@ -589,7 +582,6 @@ AccessToken OneLakeCatalog::getValidAccessToken(const CatalogState & catalog_sta std::pair OneLakeCatalog::getCurrentAccessToken() const { -<<<<<<< HEAD const auto state_snapshot = state.get(); const auto token = getValidAccessToken(*state_snapshot, /* force_update */ false); /// A token without a known expiration is reported as expiring shortly, so that the @@ -802,8 +794,6 @@ namespace AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const { -======= ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); @@ -1065,13 +1055,6 @@ BigLakeCatalog::BigLakeCatalog( DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( const CatalogState & catalog_state, bool update_token, -<<<<<<< HEAD -======= - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) bool * used_cached_oauth_token) const { /// Google Cloud OAuth2 for BigLake. @@ -1111,11 +1094,7 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } -<<<<<<< HEAD return RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); -======= - return RestCatalog::getAuthHeaders(update_token, method, url, extra_headers, body, used_cached_oauth_token); ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1139,25 +1118,6 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); -<<<<<<< HEAD -======= - - if (!google_adc_client_id.empty() && !google_adc_client_secret.empty() && !google_adc_refresh_token.empty()) - { - try - { - return retrieveGoogleCloudAccessTokenFromRefreshToken(); - } - catch (const DB::Exception & e) - { - LOG_DEBUG(log, "Failed to use ADC credentials, falling back to metadata service: {}", e.what()); - } - } - - /// Fallback to GCP metadata service (works inside GCP infrastructure) - /// https://cloud.google.com/compute/docs/metadata/overview - static constexpr auto DEFAULT_REQUEST_TOKEN_PATH = "/computeMetadata/v1/instance/service-accounts"; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) const auto & context = getContext(); @@ -1292,11 +1252,7 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { -<<<<<<< HEAD auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); -======= - auto result_headers = getAuthHeaders(update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}, &used_cached_oauth_token); ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1961,35 +1917,11 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & .create(credentials); }; -<<<<<<< HEAD - try - { - bool used_cached_oauth_token = false; - auto wb = create_buffer(false, used_cached_oauth_token); - -======= - auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) - { - DB::HTTPHeaderEntries headers = getAuthHeaders(update_token, method, url, extra_headers, body_str, &used_cached_oauth_token); - headers.emplace_back("Content-Type", "application/json"); - return DB::BuilderRWBufferFromHTTP(url) - .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) - .withMethod(method) - .withSettings(context->getReadSettings()) - .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) - .withHostFilter(&context->getRemoteHostFilter()) - .withHeaders(headers) - .withOutCallback(out_stream_callback) - .withSkipNotFound(false) - .create(credentials); - }; - try { bool used_cached_oauth_token = false; auto wb = create_buffer(false, used_cached_oauth_token); ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) String response_str; if (!ignore_result) readJSONObjectPossiblyInvalid(response_str, *wb); diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index b3f76689c182..077f186395ca 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -235,15 +235,7 @@ class RestCatalog : public ICatalog, public DB::WithContext virtual DB::HTTPHeaderEntries getAuthHeaders( const CatalogState & catalog_state, bool update_token, -<<<<<<< HEAD bool * used_cached_oauth_token) const; -======= - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -370,15 +362,7 @@ class BigLakeCatalog : public RestCatalog DB::HTTPHeaderEntries getAuthHeaders( const CatalogState & catalog_state, bool update_token, -<<<<<<< HEAD bool * used_cached_oauth_token) const override; -======= - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 52bb50da5a5c..7b1d5ff57a3a 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -262,17 +262,7 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } -<<<<<<< HEAD namespace -======= -DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( - bool /*update_token*/, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * /*used_cached_oauth_token*/) const ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { /// `signRequestWithAWSV4` returns the full set of headers that the AWS SDK kept on diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index fb7fb520bacd..2bb706df9796 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -49,7 +49,6 @@ class S3TablesCatalog final : public RestCatalog void dropTable(const String & namespace_name, const String & table_name, bool delete_data) const override; protected: -<<<<<<< HEAD /// Override the network primitives instead of `getAuthHeaders` so the SigV4 signer has /// access to the final URL, method, and request body for canonicalisation. /// `catalog_state` and `auth_headers` are unused here: authentication is derived from the @@ -67,15 +66,6 @@ class S3TablesCatalog final : public RestCatalog Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const override; -======= - DB::HTTPHeaderEntries getAuthHeaders( - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) private: const String region; diff --git a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py index 6f05d70ca9c1..d8ac831a78d8 100644 --- a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py +++ b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py @@ -490,22 +490,6 @@ def get_auth_token_profile_events(node, query_id): return refreshed, cache_hits -<<<<<<< HEAD -======= -def get_auth_token_profile_events(node, query_id): - node.query("SYSTEM FLUSH LOGS") - refreshed = int(node.query( - f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenRefreshed'] " - f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" - )) - cache_hits = int(node.query( - f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenCacheHits'] " - f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" - )) - return refreshed, cache_hits - - ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) def test_auth_token_profile_events(started_cluster): node = started_cluster.instances["node1"] @@ -515,58 +499,6 @@ def test_auth_token_profile_events(started_cluster): table_name = f"{test_ref}_table" catalog = load_catalog_impl(started_cluster) -<<<<<<< HEAD -======= - if namespace not in catalog.list_namespaces(): - catalog.create_namespace(namespace) - - schema = Schema( - NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), - NestedField(field_id=2, name="data", field_type=StringType(), required=False), - ) - catalog.create_table( - namespace + (table_name,), - schema=schema, - properties={"write.metadata.compression-codec": "none"}, - ) - - # The catalog client is initialized lazily on the first database access, - # not during CREATE DATABASE. OAuth credentials must use client_id:client_secret - # format; oauth_server_uri points to a mock token endpoint in docker compose. - create_clickhouse_iceberg_database( - started_cluster, - node, - db_name, - additional_settings={ - "catalog_credential": "test:secret", - "oauth_server_uri": MOCK_OAUTH_URL, - }, - ) - - qid1 = f"{test_ref}-show-1-{uuid.uuid4()}" - node.query(f"SHOW TABLES FROM {db_name}", query_id=qid1) - assert table_name in node.query(f"SHOW TABLES FROM {db_name}") - refreshed, cache_hits = get_auth_token_profile_events(node, qid1) - assert refreshed >= 1 - - qid2 = f"{test_ref}-show-2-{uuid.uuid4()}" - node.query(f"SHOW TABLES FROM {db_name}", query_id=qid2) - refreshed, cache_hits = get_auth_token_profile_events(node, qid2) - assert refreshed == 0 and cache_hits >= 1 - - -def test_vended_credentials_cache(started_cluster): - node = started_cluster.instances["node1"] - catalog = load_catalog_impl(started_cluster) - - test_ref = f"test_vended_credentials_cache_{uuid.uuid4().hex[:8]}" - namespace = (f"{test_ref}_namespace",) - table_name = f"{test_ref}_table" - db_name = f"{test_ref}_database" - - -def create_int_table(catalog, namespace, table_name, rows=1): ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) if namespace not in catalog.list_namespaces(): catalog.create_namespace(namespace)