From f8825a4f1a5b3e92ef0d3f99cc87640efe38eaac Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:40:53 +0200 Subject: [PATCH 1/4] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/2184 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808 Antalya 26.6: S3 tables Iceberg support + catalog profile events # Conflicts: # src/Common/ProfileEvents.cpp # src/Core/SettingsEnums.cpp # src/Core/SettingsEnums.h # src/Databases/DataLake/AWSV4Signer.cpp # src/Databases/DataLake/DatabaseDataLake.cpp # src/Databases/DataLake/GlueCatalog.cpp # src/Databases/DataLake/ICatalog.cpp # src/Databases/DataLake/RestCatalog.cpp # src/Databases/DataLake/RestCatalog.h # src/Databases/DataLake/S3TablesCatalog.cpp # src/Databases/DataLake/S3TablesCatalog.h # src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp # src/IO/S3/URI.cpp # src/IO/S3/URI.h --- src/Common/ProfileEvents.cpp | 52 ++++++ src/Core/Settings.cpp | 3 + src/Core/SettingsChangesHistory.cpp | 1 + src/Core/SettingsEnums.cpp | 4 + src/Core/SettingsEnums.h | 3 + src/Databases/DataLake/AWSV4Signer.cpp | 22 +++ src/Databases/DataLake/DatabaseDataLake.cpp | 16 ++ src/Databases/DataLake/GlueCatalog.cpp | 83 +++++++++- src/Databases/DataLake/ICatalog.cpp | 34 +++- src/Databases/DataLake/ICatalog.h | 1 - src/Databases/DataLake/RestCatalog.cpp | 144 ++++++++++++++++- src/Databases/DataLake/RestCatalog.h | 20 +++ src/Databases/DataLake/S3TablesCatalog.cpp | 153 ++++++++++++++++++ src/Databases/DataLake/S3TablesCatalog.h | 28 ++++ .../DataLake/S3TablesCredentialRefresh.cpp | 43 +++++ .../DataLake/S3TablesCredentialRefresh.h | 23 +++ src/Databases/DataLake/UnityCatalog.cpp | 48 +++++- .../tests/gtest_azure_abfss_parsing.cpp | 28 ++++ .../gtest_s3tables_credential_refresh.cpp | 112 +++++++++++++ .../enableAllExperimentalSettings.cpp | 1 + src/IO/S3/URI.cpp | 6 + src/IO/S3/URI.h | 6 + src/IO/S3/tests/gtest_s3_uri.cpp | 19 +++ 23 files changed, 833 insertions(+), 17 deletions(-) create mode 100644 src/Databases/DataLake/S3TablesCredentialRefresh.cpp create mode 100644 src/Databases/DataLake/S3TablesCredentialRefresh.h create mode 100644 src/Databases/DataLake/tests/gtest_s3tables_credential_refresh.cpp diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index b7759df73fcb..0296ff4ae577 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1659,6 +1659,7 @@ The server successfully detected this situation and will download merged part fr M(AIRowsProcessed, "Number of rows that received an AI result.", ValueType::Number) \ M(AIRowsSkipped, "Number of rows that received a default value due to quota or error.", ValueType::Number) \ \ +<<<<<<< HEAD M(StatelessWorkerRequested, "Number of stateless workers requested by queries for distributed query execution.", ValueType::Number) \ M(StatelessWorkerProvided, "Number of stateless workers provided to queries for distributed query execution.", ValueType::Number) \ M(StatelessWorkerProvisioningMicroseconds, "Total time queries spent waiting for stateless workers to be provisioned.", ValueType::Microseconds) \ @@ -1718,6 +1719,57 @@ The server successfully detected this situation and will download merged part fr M(StatelessWorkerDiscoveryHeartbeatsRejected, "Number of heartbeats the stateless worker discovery service rejected because the worker had already been evicted.", ValueType::Number) \ M(StatelessWorkerDiscoveryKeeperTransactionRetries, "Number of write transactions the stateless worker discovery service retried because its coordination store (Keeper) state was modified concurrently.", ValueType::Number) \ \ +======= + M(DataLakeRestCatalogLoadConfig, "Number of 'load config' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogLoadConfigMicroseconds, "Total time of 'load config' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetNamespaces, "Number of 'get namespaces' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetNamespacesMicroseconds, "Total time of 'get namespaces' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetTables, "Number of 'get tables' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetTableMetadata, "Number of 'get table metadata' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetTableMetadataMicroseconds, "Total time of 'get table metadata' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogGetCredentials, "Number of 'get credentials' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogCreateNamespace, "Number of 'create namespace' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogCreateNamespaceMicroseconds, "Total time of 'create namespace' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogCreateTable, "Number of 'create table' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogCreateTableMicroseconds, "Total time of 'create table' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogUpdateTable, "Number of 'update table' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogUpdateTableMicroseconds, "Total time of 'update table' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogDropTable, "Number of 'drop table' requests to Iceberg REST catalog.", ValueType::Number) \ + M(DataLakeRestCatalogDropTableMicroseconds, "Total time of 'drop table' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + \ + M(DataLakeGlueCatalogGetDatabases, "Number of 'get databases' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogGetDatabasesMicroseconds, "Total time of 'get databases' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogGetTables, "Number of 'get tables' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogGetTable, "Number of 'get table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogGetTableMicroseconds, "Total time of 'get table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogCreateDatabase, "Number of 'create database' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogCreateDatabaseMicroseconds, "Total time of 'create database' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogCreateTable, "Number of 'create table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogCreateTableMicroseconds, "Total time of 'create table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogUpdateTable, "Number of 'update table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogUpdateTableMicroseconds, "Total time of 'update table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + M(DataLakeGlueCatalogDropTable, "Number of 'drop table' requests to Iceberg Glue catalog.", ValueType::Number) \ + M(DataLakeGlueCatalogDropTableMicroseconds, "Total time of 'drop table' requests to Iceberg Glue catalog.", ValueType::Microseconds) \ + \ + M(DataLakeUnityCatalogGetTables, "Number of 'get tables' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetTablesMicroseconds, "Total time of 'get tables' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetTable, "Number of 'get table' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetTableMicroseconds, "Total time of 'get table' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetTableMetadata, "Number of 'get table metadata' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetTableMetadataMicroseconds, "Total time of 'get table metadata' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetSchemas, "Number of 'get schemas' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetSchemasMicroseconds, "Total time of 'get schemas' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + M(DataLakeUnityCatalogGetCredentials, "Number of 'get credentials' requests to Iceberg Unity catalog.", ValueType::Number) \ + M(DataLakeUnityCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ + \ + M(ObjectStorageClusterSentToMatchedReplica, "Number of tasks in ObjectStorageCluster request sent to matched replica.", ValueType::Number) \ + M(ObjectStorageClusterSentToNonMatchedReplica, "Number of tasks in ObjectStorageCluster request sent to non-matched replica.", ValueType::Number) \ + M(ObjectStorageClusterProcessedTasks, "Number of processed tasks in ObjectStorageCluster request.", ValueType::Number) \ + M(ObjectStorageClusterWaitingMicroseconds, "Time of waiting for tasks in ObjectStorageCluster request.", ValueType::Microseconds) \ +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #ifdef APPLY_FOR_EXTERNAL_EVENTS #define APPLY_FOR_EVENTS(M) APPLY_FOR_BUILTIN_EVENTS(M) APPLY_FOR_EXTERNAL_EVENTS(M) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index d0fa079ac340..f8d0ec0ea4a6 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -9097,6 +9097,9 @@ Multiple algorithms can be specified as a comma-separated list, e.g. `dphyp,gree )", EXPERIMENTAL) \ DECLARE(Bool, allow_experimental_database_paimon_rest_catalog, false, R"( Allow experimental database engine DataLakeCatalog with catalog_type = 'paimon_rest' +)", EXPERIMENTAL) \ + DECLARE(Bool, allow_experimental_database_s3_tables, false, R"( +Allow experimental database engine DataLakeCatalog with catalog_type = 's3tables' (Amazon S3 Tables Iceberg REST with SigV4) )", EXPERIMENTAL) \ DECLARE(UInt64, webassembly_udf_max_fuel, 100'000, R"( Fuel limit per WebAssembly UDF instance execution. Each WebAssembly instruction consumes some amount of fuel. The value is scaled by 1024 before being passed to the runtime, so `webassembly_udf_max_fuel = 1` corresponds to approximately 1024 fuel units. Set to 0 for no finite limit. Applies only to functions whose per-function setting `webassembly_udf_enable_fuel` is true, which is the default. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index b9acda40b208..581bd75e8d04 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -262,6 +262,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"allow_experimental_query_deduplication", false, false, "The setting is obsolete, the feature has been removed."}, {"query_plan_min_columns_for_join_lazy_indexing", 0, 3, "Control the minimum number of payload columns from the left side required for enabling lazy indexing optimization in JOIN"}, {"query_plan_max_limit_for_join_lazy_indexing", 1000, 1000, "Added new setting to control maximum limit value that allows to use query plan for lazy join indexing optimization. If zero, there is no limit"}, + {"allow_experimental_database_s3_tables", false, false, "New setting to enable experimental database S3 tables (AWS Iceberg REST catalog)."}, }); addSettingsChanges(settings_changes_history, "26.5", diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index c385e92cb493..09b18a45c21e 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -377,9 +377,13 @@ IMPLEMENT_SETTING_ENUM( {"onelake", DatabaseDataLakeCatalogType::ICEBERG_ONELAKE}, {"biglake", DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE}, {"paimon_rest", DatabaseDataLakeCatalogType::PAIMON_REST}, +<<<<<<< HEAD {"horizon", DatabaseDataLakeCatalogType::ICEBERG_HORIZON}, {"s3tables", DatabaseDataLakeCatalogType::S3_TABLES}, {"delta_sharing", DatabaseDataLakeCatalogType::ICEBERG_DELTA_SHARING}}) +======= + {"s3tables", DatabaseDataLakeCatalogType::S3_TABLES}}) +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) IMPLEMENT_SETTING_ENUM( FileCachePolicy, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index c79482c72bab..9022177313d8 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -465,8 +465,11 @@ enum class DatabaseDataLakeCatalogType : uint8_t ICEBERG_BIGLAKE, PAIMON_REST, S3_TABLES, +<<<<<<< HEAD ICEBERG_DELTA_SHARING, ICEBERG_HORIZON, +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) }; DECLARE_SETTING_ENUM(DatabaseDataLakeCatalogType) diff --git a/src/Databases/DataLake/AWSV4Signer.cpp b/src/Databases/DataLake/AWSV4Signer.cpp index 53d099bcc7c5..722862df213b 100644 --- a/src/Databases/DataLake/AWSV4Signer.cpp +++ b/src/Databases/DataLake/AWSV4Signer.cpp @@ -12,8 +12,13 @@ #include #include #include +<<<<<<< HEAD #include +======= + +#include +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include namespace DB @@ -32,6 +37,7 @@ namespace Aws::Http::HttpMethod mapPocoMethodToAws(const String & method) { +<<<<<<< HEAD using Poco::Net::HTTPRequest; static const std::pair supported_methods[] = { @@ -41,6 +47,18 @@ Aws::Http::HttpMethod mapPocoMethodToAws(const String & method) {HTTPRequest::HTTP_DELETE, Aws::Http::HttpMethod::HTTP_DELETE}, {HTTPRequest::HTTP_HEAD, Aws::Http::HttpMethod::HTTP_HEAD}, {HTTPRequest::HTTP_PATCH, Aws::Http::HttpMethod::HTTP_PATCH}, +======= + using Aws::Http::HttpMethod; + using Poco::Net::HTTPRequest; + + static const std::pair supported_methods[] = { + {HTTPRequest::HTTP_GET, HttpMethod::HTTP_GET}, + {HTTPRequest::HTTP_POST, HttpMethod::HTTP_POST}, + {HTTPRequest::HTTP_PUT, HttpMethod::HTTP_PUT}, + {HTTPRequest::HTTP_DELETE, HttpMethod::HTTP_DELETE}, + {HTTPRequest::HTTP_HEAD, HttpMethod::HTTP_HEAD}, + {HTTPRequest::HTTP_PATCH, HttpMethod::HTTP_PATCH}, +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) }; for (const auto & [poco_method, aws_method] : supported_methods) @@ -74,7 +92,11 @@ void signRequestWithAWSV4( if (!payload.empty()) { +<<<<<<< HEAD auto body_stream = Aws::MakeShared("AWSV4Signer"); +======= + auto body_stream = Aws::MakeShared("AWSV4Signer"); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) body_stream->write(payload.data(), static_cast(payload.size())); body_stream->seekg(0); request.AddContentBody(body_stream); diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index c8a17bb2c41d..49cacb16eafe 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -104,6 +104,7 @@ namespace Setting extern const SettingsBool allow_experimental_database_glue_catalog; extern const SettingsBool allow_experimental_database_hms_catalog; extern const SettingsBool allow_experimental_database_paimon_rest_catalog; + extern const SettingsBool allow_experimental_database_s3_tables; extern const SettingsBool use_hive_partitioning; extern const SettingsBool log_queries; extern const SettingsBool parallel_replicas_for_cluster_engines; @@ -426,8 +427,12 @@ void DatabaseDataLake::initialize() const url, settings[DatabaseDataLakeSetting::region].value, catalog_parameters, +<<<<<<< HEAD Context::getGlobalContextInstance(), allow_server_credentials_in_user_queries); +======= + Context::getGlobalContextInstance()); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #else throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, @@ -524,8 +529,11 @@ std::shared_ptr DatabaseDataLake::getConfigur case DatabaseDataLakeCatalogType::ICEBERG_REST: case DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE: case DatabaseDataLakeCatalogType::S3_TABLES: +<<<<<<< HEAD case DatabaseDataLakeCatalogType::ICEBERG_DELTA_SHARING: case DatabaseDataLakeCatalogType::ICEBERG_HORIZON: +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { switch (type) { @@ -1662,11 +1670,19 @@ void registerDatabaseDataLake(DatabaseFactory & factory) case DatabaseDataLakeCatalogType::S3_TABLES: { if (!args.create_query.attach +<<<<<<< HEAD && !args.context->getSettingsRef()[Setting::allow_experimental_database_iceberg]) { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "DatabaseDataLake with S3 Tables catalog (Iceberg REST) is beta. " "To allow its usage, enable setting allow_database_iceberg"); +======= + && !args.context->getSettingsRef()[Setting::allow_experimental_database_s3_tables]) + { + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "DatabaseDataLake with S3 Tables catalog is experimental. " + "To allow its usage, enable setting allow_experimental_database_s3_tables"); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } engine_func->name = "Iceberg"; diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index a2817627e33d..37f7f7c49c56 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -83,6 +84,24 @@ namespace DB::ServerSetting extern const ServerSettingsUInt64 s3_retry_attempts; } +namespace ProfileEvents +{ + extern const Event DataLakeGlueCatalogGetDatabases; + extern const Event DataLakeGlueCatalogGetDatabasesMicroseconds; + extern const Event DataLakeGlueCatalogGetTables; + extern const Event DataLakeGlueCatalogGetTablesMicroseconds; + extern const Event DataLakeGlueCatalogGetTable; + extern const Event DataLakeGlueCatalogGetTableMicroseconds; + extern const Event DataLakeGlueCatalogCreateDatabase; + extern const Event DataLakeGlueCatalogCreateDatabaseMicroseconds; + extern const Event DataLakeGlueCatalogCreateTable; + extern const Event DataLakeGlueCatalogCreateTableMicroseconds; + extern const Event DataLakeGlueCatalogUpdateTable; + extern const Event DataLakeGlueCatalogUpdateTableMicroseconds; + extern const Event DataLakeGlueCatalogDropTable; + extern const Event DataLakeGlueCatalogDropTableMicroseconds; +} + namespace CurrentMetrics { extern const Metric MarkCacheBytes; @@ -212,7 +231,14 @@ DataLake::ICatalog::Namespaces GlueCatalog::getDatabases(const std::string & pre do { request.SetNextToken(next_token); - auto outcome = glue_client->GetDatabases(request); + + Aws::Glue::Model::GetDatabasesOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetDatabases); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetDatabasesMicroseconds); + outcome = glue_client->GetDatabases(request); + } + if (outcome.IsSuccess()) { const auto & databases_result = outcome.GetResult(); @@ -261,7 +287,12 @@ CatalogTables GlueCatalog::getTablesForDatabase(const std::string & db_name, siz do { request.SetNextToken(next_token); - auto outcome = glue_client->GetTables(request); + Aws::Glue::Model::GetTablesOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTables); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTablesMicroseconds); + outcome = glue_client->GetTables(request); + } if (outcome.IsSuccess()) { const auto & tables_result = outcome.GetResult(); @@ -326,8 +357,19 @@ CatalogTables GlueCatalog::listTablesInNamespaceDirect(const std::string & names bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name) const { +<<<<<<< HEAD TableMetadata metadata; return tryGetTableMetadata(database_name, table_name, metadata); +======= + Aws::Glue::Model::GetTableRequest request; + request.SetDatabaseName(database_name); + request.SetName(table_name); + + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); + auto outcome = glue_client->GetTable(request); + return outcome.IsSuccess(); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } bool GlueCatalog::tryGetTableMetadata( @@ -339,7 +381,12 @@ bool GlueCatalog::tryGetTableMetadata( request.SetDatabaseName(database_name); request.SetName(table_name); - auto outcome = glue_client->GetTable(request); + Aws::Glue::Model::GetTableOutcome outcome; + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); + outcome = glue_client->GetTable(request); + } if (outcome.IsSuccess()) { const auto & table_outcome = outcome.GetResult().GetTable(); @@ -635,6 +682,7 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons db_input.SetName(namespace_name); create_request.SetDatabaseInput(db_input); +<<<<<<< HEAD auto outcome = glue_client->CreateDatabase(create_request); if (!outcome.IsSuccess() && outcome.GetError().GetErrorType() != Aws::Glue::GlueErrors::ALREADY_EXISTS) { @@ -643,6 +691,11 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons "Exception calling CreateDatabase for namespace {}: {}", namespace_name, outcome.GetError().GetMessage()); } +======= + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); + glue_client->CreateDatabase(create_request); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*metadata_content*/) const @@ -672,7 +725,13 @@ void GlueCatalog::createTable(const String & namespace_name, const String & tabl request.SetTableInput(table_input); - auto response = glue_client->CreateTable(request); + Aws::Glue::Model::CreateTableOutcome response; + + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateTableMicroseconds); + response = glue_client->CreateTable(request); + } if (!response.IsSuccess()) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Can not create metadata in glue catalog: {}", response.GetError().GetMessage()); @@ -707,7 +766,13 @@ bool GlueCatalog::updateMetadata(const String & namespace_name, const String & t request.SetTableInput(table_input); - auto response = glue_client->UpdateTable(request); + Aws::Glue::Model::UpdateTableOutcome response; + + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogUpdateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogUpdateTableMicroseconds); + response = glue_client->UpdateTable(request); + } if (!response.IsSuccess()) throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Can not update metadata in glue catalog {}", response.GetError().GetMessage()); @@ -731,7 +796,13 @@ void GlueCatalog::dropTable(const String & namespace_name, const String & table_ request.SetDatabaseName(namespace_name); request.SetName(table_name); - auto response = glue_client->DeleteTable(request); + Aws::Glue::Model::DeleteTableOutcome response; + + { + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogDropTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogDropTableMicroseconds); + response = glue_client->DeleteTable(request); + } if (!response.IsSuccess()) throw DB::Exception( diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index cdf297f0671e..f989b42c795c 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -113,6 +113,7 @@ void TableMetadata::setLocation(const std::string & location_) auto pos_to_bucket = pos + std::strlen("://"); auto path_separator_offset = location_.substr(pos_to_bucket).find('/'); +<<<<<<< HEAD size_t pos_to_path = 0; if (path_separator_offset == std::string::npos) { @@ -125,6 +126,21 @@ void TableMetadata::setLocation(const std::string & location_) throw DB::Exception( DB::ErrorCodes::NOT_IMPLEMENTED, "Location without a path is only supported for the s3 scheme: {}", location_); +======= + if (pos_to_path == std::string::npos) + { + /// An empty path is allowed for AWS S3 Tables: the table location is just `s3://`. + if (storage_type_str == "s3://") + { + location_without_path = location_; + path.clear(); + bucket = location_.substr(pos_to_bucket); + return; + } + + throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected location format: {}", location_); + } +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) pos_to_path = location_.size(); location_without_path = location_; @@ -320,8 +336,22 @@ std::string TableMetadata::getMetadataLocation(const std::string & iceberg_metad metadata_location = metadata_location.substr(storage_type_str.size()); if (data_location.starts_with(storage_type_str)) data_location = data_location.substr(storage_type_str.size()); - else if (!endpoint.empty() && data_location.starts_with(endpoint)) - data_location = data_location.substr(endpoint.size()); + else if (!endpoint.empty()) + { + std::string normalized_endpoint = endpoint; + if (normalized_endpoint.ends_with('/')) + normalized_endpoint.pop_back(); + + if (data_location.starts_with(normalized_endpoint)) + { + data_location = data_location.substr(normalized_endpoint.size()); + /// `metadata_location` is relative to the bucket (the `s3://` prefix is stripped above), + /// while `data_location` still has the leading slash left over from the endpoint, + /// e.g. "/bucket/table-uuid/". Drop it so that the prefix comparison below works. + if (azure_account_with_suffix.empty() && data_location.starts_with('/')) + data_location = data_location.substr(1); + } + } if (metadata_location.starts_with(data_location)) { diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index e79ebf686f68..57a2adb713b5 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -64,7 +64,6 @@ class TableMetadata bool hasLocation() const; bool hasSchema() const; bool hasStorageCredentials() const; - bool hasDataLakeSpecificProperties() const; void setLocation(const std::string & location_); std::string getLocation() const; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 3b3b86785c5a..562246f9026b 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -77,6 +78,7 @@ namespace DB::FailPoints namespace ProfileEvents { +<<<<<<< HEAD extern const Event OneLakeAccessTokenRequests; extern const Event OneLakeAccessTokenRequestFailures; extern const Event OneLakeAccessTokenRequestMicroseconds; @@ -92,6 +94,26 @@ namespace DB::DatabaseDataLakeSetting extern const DatabaseDataLakeSettingsString onelake_refresh_token; extern const DatabaseDataLakeSettingsString onelake_client_id; extern const DatabaseDataLakeSettingsString onelake_client_secret; +======= + extern const Event DataLakeRestCatalogLoadConfig; + extern const Event DataLakeRestCatalogLoadConfigMicroseconds; + extern const Event DataLakeRestCatalogGetNamespaces; + extern const Event DataLakeRestCatalogGetNamespacesMicroseconds; + extern const Event DataLakeRestCatalogGetTables; + extern const Event DataLakeRestCatalogGetTablesMicroseconds; + extern const Event DataLakeRestCatalogGetTableMetadata; + extern const Event DataLakeRestCatalogGetTableMetadataMicroseconds; + extern const Event DataLakeRestCatalogGetCredentials; + extern const Event DataLakeRestCatalogGetCredentialsMicroseconds; + extern const Event DataLakeRestCatalogCreateNamespace; + extern const Event DataLakeRestCatalogCreateNamespaceMicroseconds; + extern const Event DataLakeRestCatalogCreateTable; + extern const Event DataLakeRestCatalogCreateTableMicroseconds; + extern const Event DataLakeRestCatalogUpdateTable; + extern const Event DataLakeRestCatalogUpdateTableMicroseconds; + extern const Event DataLakeRestCatalogDropTable; + extern const Event DataLakeRestCatalogDropTableMicroseconds; +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } namespace DataLake @@ -248,10 +270,19 @@ RestCatalog::RestCatalog( RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers) { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; +<<<<<<< HEAD auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */ {}, auth_headers); +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::string json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); + + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); + auto buf = createReadBuffer(CONFIG_ENDPOINT, params); + readJSONObjectPossiblyInvalid(json_str, *buf); + } LOG_DEBUG(log, "Received catalog configuration settings: {}", json_str); @@ -295,7 +326,16 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } +<<<<<<< HEAD DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const +======= +DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( + bool update_token, + const String & /*method*/, + const Poco::URI & /*url*/, + const DB::HTTPHeaderEntries & /*extra_headers*/, + const String & /*body*/) const +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -997,7 +1037,16 @@ BigLakeCatalog::BigLakeCatalog( state.set(std::make_unique(std::move(initial_state))); } +<<<<<<< HEAD DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const +======= +DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( + bool update_token, + const String & /*method*/, + const Poco::URI & /*url*/, + const DB::HTTPHeaderEntries & /*extra_headers*/, + const String & /*body*/) const +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -1184,7 +1233,11 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token) { +<<<<<<< HEAD auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token); +======= + auto result_headers = getAuthHeaders(update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1380,8 +1433,14 @@ RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & bas if (!page_token.empty()) params.push_back({"pageToken", page_token}); +<<<<<<< HEAD auto buf = createReadBuffer( *state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params, /* headers */ {}, /* auth_headers */ std::nullopt); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); + auto buf = createReadBuffer(config.prefix / NAMESPACES_ENDPOINT, params); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1531,8 +1590,14 @@ DB::Names RestCatalog::listTablesInNamespace(const std::string & base_namespace, if (!page_token.empty()) params.push_back({"pageToken", page_token}); +<<<<<<< HEAD auto buf = createReadBuffer( *state_snapshot, state_snapshot->config.prefix / endpoint, params, /* headers */ {}, /* auth_headers */ std::nullopt); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); + auto buf = createReadBuffer(config.prefix / endpoint, params); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -1677,6 +1742,7 @@ bool RestCatalog::getTableMetadataImpl( const auto state_snapshot = state.get(); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; +<<<<<<< HEAD auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); if (buf->eof()) @@ -1685,8 +1751,23 @@ bool RestCatalog::getTableMetadataImpl( return false; } +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) String json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); + + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); + auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + + if (buf->eof()) + { + LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); + return false; + } + + readJSONObjectPossiblyInvalid(json_str, *buf); + } #ifdef DEBUG_OR_SANITIZER_BUILD /// This log message might contain credentials, @@ -1761,12 +1842,15 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & request_body->stringify(oss); const std::string body_str = DB::removeEscapedSlashes(oss.str()); +<<<<<<< HEAD LOG_TEST(log, "REST catalog {} {} body ({} bytes): {}", method, endpoint, body_str.size(), body_str); DB::HTTPHeaderEntries headers = getAuthHeaders(catalog_state, /* update_token = */ true); headers.emplace_back("Content-Type", "application/json"); headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) const auto & context = getContext(); DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; @@ -1781,6 +1865,14 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & /// enable_url_encoding=false to allow using tables with encoded sequences in names like 'foo%2Fbar' Poco::URI url(endpoint, /* enable_url_encoding */ false); +<<<<<<< HEAD +======= + DB::HTTPHeaderEntries extra_headers; + extra_headers.emplace_back("Content-Type", "application/json"); + + DB::HTTPHeaderEntries headers = getAuthHeaders(/* update_token = */ true, method, url, extra_headers, body_str); + headers.emplace_back("Content-Type", "application/json"); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withMethod(method) @@ -1837,7 +1929,13 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons try { +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); + sendRequest(endpoint, request_body); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } catch (const DB::HTTPException & e) { @@ -1881,7 +1979,13 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl try { +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); + sendRequest(endpoint, request_body); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } catch (const DB::HTTPException & ex) { @@ -1948,7 +2052,13 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t try { +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); + sendRequest(endpoint, request_body); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } catch (const DB::HTTPException & ex) { @@ -2034,13 +2144,25 @@ bool RestCatalog::updateSchema( void RestCatalog::dropTable(const String & namespace_name, const String & table_name, bool /*delete_data*/) const { +<<<<<<< HEAD const auto state_snapshot = state.get(); const std::string endpoint = fmt::format("{}/namespaces/{}/tables/{}?purgeRequested=False", base_url, namespace_name, table_name); +======= + const std::string endpoint + = (base_url / config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string() + + "?purgeRequested=False"; +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) Poco::JSON::Object::Ptr request_body = nullptr; try { +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); + sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } catch (const DB::HTTPException & ex) { @@ -2128,6 +2250,7 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; +<<<<<<< HEAD auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); if (buf->eof()) @@ -2136,8 +2259,23 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal return nullptr; } +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) String json_str; - readJSONObjectPossiblyInvalid(json_str, *buf); + + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); + auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + + if (buf->eof()) + { + LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); + return nullptr; + } + + readJSONObjectPossiblyInvalid(json_str, *buf); + } Poco::JSON::Parser parser; Poco::Dynamic::Var json = parser.parse(json_str); diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index a21474fd5081..ee473cc6d0cf 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -205,11 +205,22 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, TableMetadata & result) const; +<<<<<<< HEAD /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); virtual DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const; +======= + Config loadConfig(); + virtual DB::HTTPHeaderEntries getAuthHeaders( + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}) const; +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; + static void parseCatalogConfigurationSettings(const Poco::JSON::Object::Ptr & object, Config & result); virtual void sendRequest( @@ -325,7 +336,16 @@ class BigLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE; } +<<<<<<< HEAD DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const override; +======= + DB::HTTPHeaderEntries getAuthHeaders( + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}) const override; +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index bbf1365250b5..a86467bdaadf 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -4,10 +4,18 @@ #include #include +<<<<<<< HEAD +======= +#include +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include #include +<<<<<<< HEAD +======= +#include +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include #include @@ -17,35 +25,53 @@ #include #include #include +<<<<<<< HEAD #include #include #include #include #include +======= +#include +#include +#include +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include #include #include +<<<<<<< HEAD #include +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) namespace DB::ErrorCodes { extern const int BAD_ARGUMENTS; extern const int DATALAKE_DATABASE_ERROR; +<<<<<<< HEAD extern const int SUPPORT_IS_DISABLED; +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } namespace DB::Setting { +<<<<<<< HEAD extern const SettingsUInt64 s3_max_connections; +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) extern const SettingsUInt64 s3_max_redirects; extern const SettingsUInt64 s3_retry_attempts; extern const SettingsBool s3_slow_all_threads_after_network_error; extern const SettingsBool enable_s3_requests_logging; +<<<<<<< HEAD extern const SettingsUInt64 s3_connect_timeout_ms; extern const SettingsUInt64 s3_request_timeout_ms; +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } namespace DB::ServerSetting @@ -54,6 +80,15 @@ namespace DB::ServerSetting extern const ServerSettingsUInt64 s3_retry_attempts; } +<<<<<<< HEAD +======= +namespace ProfileEvents +{ + extern const Event DataLakeRestCatalogDropTable; + extern const Event DataLakeRestCatalogDropTableMicroseconds; +} + +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) namespace DataLake { @@ -62,12 +97,19 @@ S3TablesCatalog::S3TablesCatalog( const String & base_url_, const String & region_, const CatalogSettings & catalog_settings_, +<<<<<<< HEAD DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_) : RestCatalog(warehouse_, base_url_, "", "", false, context_) , region(region_) , storage_endpoint(catalog_settings_.storage_endpoint) , signing_service("s3tables") +======= + DB::ContextPtr context_) + : RestCatalog(warehouse_, base_url_, "", "", false, context_) + , region(region_) + , storage_endpoint(catalog_settings_.storage_endpoint) +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { if (region.empty()) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "S3 Tables catalog requires non-empty `region` setting"); @@ -77,12 +119,15 @@ S3TablesCatalog::S3TablesCatalog( creds_config.role_arn = catalog_settings_.aws_role_arn; creds_config.role_session_name = catalog_settings_.aws_role_session_name; +<<<<<<< HEAD /// An S3 Tables catalog is created by user SQL, so it must not reuse the server's own credentials unless /// that was allowed at CREATE time. The cached catalog holds the global context, whose live setting never /// reflects the creating session, so pass the value captured then instead. creds_config.forbid_implicit_credentials = getContext()->shouldRestrictUserQueryS3Credentials(allow_server_credentials_in_user_queries_); +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) const auto & server_settings = getContext()->getGlobalContext()->getServerSettings(); const DB::Settings & global_settings = getContext()->getGlobalContext()->getSettingsRef(); @@ -120,6 +165,7 @@ S3TablesCatalog::S3TablesCatalog( Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, /* urlEscapePath = */ false); +<<<<<<< HEAD /// The signer is fully initialised above, so the virtual `createReadBuffer` reached by /// `loadConfig` dispatches to this class's SigV4 implementation. CatalogState initial_state; @@ -141,6 +187,23 @@ S3TablesCatalog::S3TablesCatalog( CatalogTables S3TablesCatalog::getTables() const { auto namespaces = listChildNamespaces(""); +======= + config = loadConfig(); + + if (config.prefix.empty()) + { + String encoded_warehouse; + Poco::URI::encode(warehouse_, "", encoded_warehouse); + config.prefix = encoded_warehouse; + } +} + +/// S3 Tables only supports a single level of namespaces (no nesting), +/// so we use flat getNamespaces() instead of the base class's getNamespacesRecursive(). +DB::Names S3TablesCatalog::getTables() const +{ + auto namespaces = getNamespaces(""); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, DB::ThreadName::DATALAKE_REST_CATALOG); @@ -152,12 +215,17 @@ CatalogTables S3TablesCatalog::getTables() const runner.enqueueAndKeepTrack( [&, ns] { +<<<<<<< HEAD auto tables_in_ns = listTablesInNamespace(ns); +======= + auto tables_in_ns = RestCatalog::getTables(ns); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::lock_guard lock(mutex); std::move(tables_in_ns.begin(), tables_in_ns.end(), std::back_inserter(tables)); }); } runner.waitForAllToFinishAndRethrowFirstError(); +<<<<<<< HEAD /// A REST catalog is Iceberg-only, so every listed table is readable. CatalogTables result; @@ -165,6 +233,9 @@ CatalogTables S3TablesCatalog::getTables() const for (auto & name : tables) result.push_back(CatalogTable{.name = std::move(name)}); return result; +======= + return tables; +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } bool S3TablesCatalog::tryGetTableMetadata( @@ -175,6 +246,7 @@ bool S3TablesCatalog::tryGetTableMetadata( if (!RestCatalog::tryGetTableMetadata(namespace_name, table_name, result)) return false; +<<<<<<< HEAD /// For S3 Tables the catalog and the underlying data live in AWS S3 under the same /// AWS principal, so endpoint/metadata-location normalization and fallback IAM /// credential injection must always run - even when the engine was created with @@ -191,6 +263,15 @@ bool S3TablesCatalog::tryGetTableMetadata( if (result.hasStorageCredentials()) { auto creds = std::dynamic_pointer_cast(result.getStorageCredentials()); +======= + if (!result.requiresCredentials()) + return true; + + bool need_credentials = true; + if (const auto storage_credentials = result.getStorageCredentials()) + { + auto creds = std::dynamic_pointer_cast(storage_credentials); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) if (creds && !creds->isEmpty()) need_credentials = false; } @@ -199,7 +280,16 @@ bool S3TablesCatalog::tryGetTableMetadata( { LOG_DEBUG(log, "S3 Tables: no vended credentials for {}.{}, injecting catalog IAM credentials", namespace_name, table_name); auto aws_creds = credentials_provider->GetAWSCredentials(); +<<<<<<< HEAD result.withStorageCredentials(); +======= + if (aws_creds.GetAWSAccessKeyId().empty() || aws_creds.GetAWSSecretKey().empty()) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "S3 Tables: catalog IAM credentials are empty for {}.{}, " + "check AWS credentials configuration", + namespace_name, table_name); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) result.setStorageCredentials(std::make_shared( aws_creds.GetAWSAccessKeyId(), aws_creds.GetAWSSecretKey(), aws_creds.GetSessionToken())); } @@ -207,12 +297,17 @@ bool S3TablesCatalog::tryGetTableMetadata( if (result.getEndpoint().empty()) { String endpoint = storage_endpoint.empty() +<<<<<<< HEAD ? DB::S3::expandRegionToAmazonPath(region) +======= + ? DB::S3::resolveS3Endpoint(region) +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) : storage_endpoint; LOG_DEBUG(log, "S3 Tables: no endpoint for {}.{}, injecting: {}", namespace_name, table_name, endpoint); result.setEndpoint(endpoint); } +<<<<<<< HEAD if (auto props = result.getDataLakeSpecificProperties(); props && !props->iceberg_metadata_file_location.empty()) { @@ -243,24 +338,63 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta const auto state_snapshot = state.get(); const std::string endpoint = (base_url / state_snapshot->config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() +======= + return true; +} + +ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) +{ + auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id); + return [this, base_callback = std::move(base_cb)] () -> std::shared_ptr + { + if (base_callback) + { + if (auto creds = (*base_callback)()) + { + auto s3_creds = std::dynamic_pointer_cast(creds); + if (s3_creds && !s3_creds->isEmpty()) + return creds; + } + LOG_DEBUG(log, "S3 Tables: vended credentials unavailable on refresh, falling back to catalog IAM credentials"); + } + + return resolveS3TablesRefreshCredentials(std::nullopt, *credentials_provider); + }; +} + +void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name) const +{ + const std::string endpoint + = (base_url / config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + "?purgeRequested=True"; Poco::JSON::Object::Ptr request_body = nullptr; try { +<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); LOG_INFO(log, "S3 Tables: dropped table {}.{} (purgeRequested=True)", namespace_name, table_name); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); + sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } catch (const DB::HTTPException & ex) { if (ex.getHTTPStatus() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) +<<<<<<< HEAD // 404 is returned by the API when the table does not exist +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) LOG_DEBUG(log, "S3 Tables: table {}.{} already does not exist (404 on purge-delete)", namespace_name, table_name); else throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Failed to drop table {}", ex.displayText()); } } +<<<<<<< HEAD namespace { @@ -284,6 +418,22 @@ DB::HTTPHeaderEntries extractSigV4AuthHeaders(DB::HTTPHeaderEntries && all_signe for (auto & h : all_signed) { if (isSigV4AuthHeader(h.name)) +======= +DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( + bool /*update_token*/, + const String & method, + const Poco::URI & url, + const DB::HTTPHeaderEntries & extra_headers, + const String & body) const +{ + DB::HTTPHeaderEntries all_signed; + signRequestWithAWSV4(method, url, extra_headers, body, *signer, region, "s3tables", all_signed); + + DB::HTTPHeaderEntries auth_headers; + for (auto & h : all_signed) + { + if (h.name == "authorization" || h.name.starts_with("x-amz-")) +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) auth_headers.push_back(std::move(h)); } return auth_headers; @@ -291,6 +441,7 @@ DB::HTTPHeaderEntries extractSigV4AuthHeaders(DB::HTTPHeaderEntries && all_signe } +<<<<<<< HEAD DB::ReadWriteBufferFromHTTPPtr S3TablesCatalog::createReadBuffer( const CatalogState & /* catalog_state */, const std::string & endpoint, @@ -379,4 +530,6 @@ void S3TablesCatalog::sendRequest( } +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #endif diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index 65ca66500ed1..a2befada02c2 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -7,8 +7,11 @@ #include #include +<<<<<<< HEAD #include +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include @@ -31,6 +34,7 @@ class S3TablesCatalog final : public RestCatalog const String & base_url_, const String & region_, const DataLake::CatalogSettings & catalog_settings_, +<<<<<<< HEAD DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_); @@ -39,12 +43,20 @@ class S3TablesCatalog final : public RestCatalog CatalogTables getTables() const override; bool managesTableLocation() const override { return true; } +======= + DB::ContextPtr context_); + + DB::DatabaseDataLakeCatalogType getCatalogType() const override { return DB::DatabaseDataLakeCatalogType::S3_TABLES; } + + DB::Names getTables() const override; +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) bool tryGetTableMetadata( const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const override; +<<<<<<< HEAD void dropTable(const String & namespace_name, const String & table_name, bool delete_data) const override; protected: @@ -65,11 +77,27 @@ class S3TablesCatalog final : public RestCatalog Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const override; +======= + void dropTable(const String & namespace_name, const String & table_name) const override; + + ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; + +protected: + DB::HTTPHeaderEntries getAuthHeaders( + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}) const override; +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) private: const String region; const String storage_endpoint; +<<<<<<< HEAD const String signing_service; +======= +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::shared_ptr credentials_provider; std::unique_ptr signer; }; diff --git a/src/Databases/DataLake/S3TablesCredentialRefresh.cpp b/src/Databases/DataLake/S3TablesCredentialRefresh.cpp new file mode 100644 index 000000000000..8de09e4d553e --- /dev/null +++ b/src/Databases/DataLake/S3TablesCredentialRefresh.cpp @@ -0,0 +1,43 @@ +#include "config.h" + +#if USE_AVRO && USE_SSL && USE_AWS_S3 + +#include + +namespace DataLake +{ + +namespace +{ + +std::shared_ptr getCatalogIAMCredentials(Aws::Auth::AWSCredentialsProvider & provider) +{ + auto aws_creds = provider.GetAWSCredentials(); + if (aws_creds.GetAWSAccessKeyId().empty() || aws_creds.GetAWSSecretKey().empty()) + return nullptr; + return std::make_shared( + aws_creds.GetAWSAccessKeyId(), aws_creds.GetAWSSecretKey(), aws_creds.GetSessionToken()); +} + +} + +std::shared_ptr resolveS3TablesRefreshCredentials( + const ICatalog::CredentialsRefreshCallback & base_callback, + Aws::Auth::AWSCredentialsProvider & credentials_provider) +{ + if (base_callback) + { + if (auto creds = (*base_callback)()) + { + auto s3_creds = std::dynamic_pointer_cast(creds); + if (s3_creds && !s3_creds->isEmpty()) + return creds; + } + } + + return getCatalogIAMCredentials(credentials_provider); +} + +} + +#endif diff --git a/src/Databases/DataLake/S3TablesCredentialRefresh.h b/src/Databases/DataLake/S3TablesCredentialRefresh.h new file mode 100644 index 000000000000..e5b959f88860 --- /dev/null +++ b/src/Databases/DataLake/S3TablesCredentialRefresh.h @@ -0,0 +1,23 @@ +#pragma once + +#include "config.h" + +#if USE_AVRO && USE_SSL && USE_AWS_S3 + +#include +#include + +#include + +#include + +namespace DataLake +{ + +std::shared_ptr resolveS3TablesRefreshCredentials( + const ICatalog::CredentialsRefreshCallback & base_callback, + Aws::Auth::AWSCredentialsProvider & credentials_provider); + +} + +#endif diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 2d78df3ec7d8..da9ece058891 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,20 @@ #include #include +namespace ProfileEvents +{ + extern const Event DataLakeUnityCatalogGetTables; + extern const Event DataLakeUnityCatalogGetTablesMicroseconds; + extern const Event DataLakeUnityCatalogGetTable; + extern const Event DataLakeUnityCatalogGetTableMicroseconds; + extern const Event DataLakeUnityCatalogGetTableMetadata; + extern const Event DataLakeUnityCatalogGetTableMetadataMicroseconds; + extern const Event DataLakeUnityCatalogGetCredentials; + extern const Event DataLakeUnityCatalogGetCredentialsMicroseconds; + extern const Event DataLakeUnityCatalogGetSchemas; + extern const Event DataLakeUnityCatalogGetSchemasMicroseconds; +} + namespace DB::ErrorCodes { extern const int DATALAKE_DATABASE_ERROR; @@ -139,7 +154,14 @@ Poco::JSON::Object::Ptr UnityCatalog::requestReadCredentials(const String & tabl request_body.set("operation", "READ"); auto callback = [&request_body] (std::ostream & os) { request_body.stringify(os); }; - auto [json, _] = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); + + Poco::Dynamic::Var json; + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetCredentials); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetCredentialsMicroseconds); + std::string _; + std::tie(json, _) = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); + } return json.extract(); } @@ -200,7 +222,11 @@ bool UnityCatalog::tryGetTableMetadata( std::string json_str; try { - std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / full_table_name); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetTableMetadata); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetTableMetadataMicroseconds); + std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / full_table_name); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (hasValueAndItsNotNone("name", object) && object->get("name").extract() == table_name) { @@ -321,7 +347,11 @@ bool UnityCatalog::existsTable(const std::string & schema_name, const std::strin Poco::Dynamic::Var json; try { - std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / (warehouse + "." + schema_name + "." + table_name)); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetTable); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetTableMicroseconds); + std::tie(json, json_str) = getJSONRequest(std::filesystem::path{TABLES_ENDPOINT} / (warehouse + "." + schema_name + "." + table_name)); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (hasValueAndItsNotNone("name", object) && object->get("name").extract() == table_name) return true; @@ -349,7 +379,11 @@ CatalogTables UnityCatalog::getTablesForSchema(const std::string & schema, size_ try { - std::tie(json, json_str) = getJSONRequest(TABLES_ENDPOINT, params); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetTables); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetTablesMicroseconds); + std::tie(json, json_str) = getJSONRequest(TABLES_ENDPOINT, params); + } const Poco::JSON::Object::Ptr & object = json.extract(); if (!hasValueAndItsNotNone("tables", object)) @@ -416,7 +450,11 @@ DataLake::ICatalog::Namespaces UnityCatalog::getSchemas(const std::string & base try { - std::tie(json, json_str) = getJSONRequest(SCHEMAS_ENDPOINT, params); + { + ProfileEvents::increment(ProfileEvents::DataLakeUnityCatalogGetSchemas); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeUnityCatalogGetSchemasMicroseconds); + std::tie(json, json_str) = getJSONRequest(SCHEMAS_ENDPOINT, params); + } const Poco::JSON::Object::Ptr & object = json.extract(); auto schemas_object = object->get("schemas").extract(); diff --git a/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp b/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp index cc80f91b7382..a8b4703ed8ac 100644 --- a/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp +++ b/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp @@ -240,6 +240,7 @@ TEST_F(AzureAbfssParsingTest, TableMetadataGetLocationWithEndpointVirtualHostedD EXPECT_EQ(location, "https://my.dotted.bucket.s3.mycompany.com/path/to/table/"); } +<<<<<<< HEAD TEST_F(AzureAbfssParsingTest, TableMetadataAbfssEndpointAlreadyContainsContainerDefault) { TableMetadata metadata; @@ -282,6 +283,33 @@ TEST_F(AzureAbfssParsingTest, TableMetadataS3EndpointAlreadyEndsWithBucketForceA metadata.setEndpoint("http://minio:9000/warehouse-rest"); EXPECT_EQ(metadata.getLocation(), "http://minio:9000/warehouse-rest/warehouse-rest/data/testns/testtable/"); +======= +TEST_F(AzureAbfssParsingTest, TableMetadataGetMetadataLocationS3TablesWithAwsEndpoint) +{ + TableMetadata metadata; + metadata.withLocation(); + metadata.setLocation("s3://bucket/table-uuid/"); + metadata.setEndpoint("https://s3.us-east-2.amazonaws.com"); + + EXPECT_EQ(metadata.getLocation(), "https://s3.us-east-2.amazonaws.com/bucket/table-uuid/"); + + const std::string metadata_file = + "s3://bucket/table-uuid/metadata/v1.metadata.json"; + EXPECT_EQ(metadata.getMetadataLocation(metadata_file), "metadata/v1.metadata.json"); +} + +TEST_F(AzureAbfssParsingTest, TableMetadataGetMetadataLocationS3TablesEmptyPathWithAwsEndpoint) +{ + TableMetadata metadata; + metadata.withLocation(); + metadata.setLocation("s3://bucket"); + metadata.setEndpoint("https://s3.us-east-2.amazonaws.com"); + + EXPECT_EQ(metadata.getLocation(), "https://s3.us-east-2.amazonaws.com/bucket/"); + + const std::string metadata_file = "s3://bucket/metadata/v1.metadata.json"; + EXPECT_EQ(metadata.getMetadataLocation(metadata_file), "metadata/v1.metadata.json"); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } } diff --git a/src/Databases/DataLake/tests/gtest_s3tables_credential_refresh.cpp b/src/Databases/DataLake/tests/gtest_s3tables_credential_refresh.cpp new file mode 100644 index 000000000000..6c5bc705fea5 --- /dev/null +++ b/src/Databases/DataLake/tests/gtest_s3tables_credential_refresh.cpp @@ -0,0 +1,112 @@ +#include "config.h" + +#if USE_AVRO && USE_SSL && USE_AWS_S3 + +#include + +#include +#include + +#include +#include + +#include + +namespace +{ + +class RotatingAWSCredentialsProvider : public Aws::Auth::AWSCredentialsProvider +{ +public: + explicit RotatingAWSCredentialsProvider(std::vector credentials_sets_) + : credentials_sets(std::move(credentials_sets_)) + { + } + + Aws::Auth::AWSCredentials GetAWSCredentials() override + { + std::lock_guard lock(mutex); + const size_t index = call_count++; + if (index >= credentials_sets.size()) + return credentials_sets.back(); + return credentials_sets[index]; + } + +private: + std::vector credentials_sets; + std::mutex mutex; + size_t call_count = 0; +}; + +} + +TEST(S3TablesCredentialRefresh, FallsBackToCatalogIAMWhenVendedCredentialsMissing) +{ + RotatingAWSCredentialsProvider provider({ + Aws::Auth::AWSCredentials("access_key_1", "secret_key_1", "session_token_1"), + Aws::Auth::AWSCredentials("access_key_2", "secret_key_2", "session_token_2"), + }); + + DataLake::ICatalog::CredentialsRefreshCallback base_callback = []() -> std::shared_ptr + { + return nullptr; + }; + + auto first = DataLake::resolveS3TablesRefreshCredentials(base_callback, provider); + ASSERT_NE(first, nullptr); + auto first_s3 = std::dynamic_pointer_cast(first); + ASSERT_NE(first_s3, nullptr); + EXPECT_EQ(first_s3->getAccessKeyId(), "access_key_1"); + EXPECT_EQ(first_s3->getSecretAccessKey(), "secret_key_1"); + EXPECT_EQ(first_s3->getSessionToken(), "session_token_1"); + + auto second = DataLake::resolveS3TablesRefreshCredentials(base_callback, provider); + ASSERT_NE(second, nullptr); + auto second_s3 = std::dynamic_pointer_cast(second); + ASSERT_NE(second_s3, nullptr); + EXPECT_EQ(second_s3->getAccessKeyId(), "access_key_2"); + EXPECT_EQ(second_s3->getSecretAccessKey(), "secret_key_2"); + EXPECT_EQ(second_s3->getSessionToken(), "session_token_2"); +} + +TEST(S3TablesCredentialRefresh, PrefersVendedCredentialsWhenPresent) +{ + RotatingAWSCredentialsProvider provider({ + Aws::Auth::AWSCredentials("catalog_access", "catalog_secret", "catalog_token"), + }); + + DataLake::ICatalog::CredentialsRefreshCallback base_callback = []() -> std::shared_ptr + { + return std::make_shared("vended_access", "vended_secret", "vended_token"); + }; + + auto creds = DataLake::resolveS3TablesRefreshCredentials(base_callback, provider); + ASSERT_NE(creds, nullptr); + auto s3_creds = std::dynamic_pointer_cast(creds); + ASSERT_NE(s3_creds, nullptr); + EXPECT_EQ(s3_creds->getAccessKeyId(), "vended_access"); + EXPECT_EQ(s3_creds->getSecretAccessKey(), "vended_secret"); + EXPECT_EQ(s3_creds->getSessionToken(), "vended_token"); +} + +TEST(S3TablesCredentialRefresh, FallsBackWhenVendedCredentialsEmpty) +{ + RotatingAWSCredentialsProvider provider({ + Aws::Auth::AWSCredentials("catalog_access", "catalog_secret", "catalog_token"), + }); + + DataLake::ICatalog::CredentialsRefreshCallback base_callback = []() -> std::shared_ptr + { + return std::make_shared("", "", ""); + }; + + auto creds = DataLake::resolveS3TablesRefreshCredentials(base_callback, provider); + ASSERT_NE(creds, nullptr); + auto s3_creds = std::dynamic_pointer_cast(creds); + ASSERT_NE(s3_creds, nullptr); + EXPECT_EQ(s3_creds->getAccessKeyId(), "catalog_access"); + EXPECT_EQ(s3_creds->getSecretAccessKey(), "catalog_secret"); + EXPECT_EQ(s3_creds->getSessionToken(), "catalog_token"); +} + +#endif diff --git a/src/Databases/enableAllExperimentalSettings.cpp b/src/Databases/enableAllExperimentalSettings.cpp index dde51a54f9a2..6878ee5ab646 100644 --- a/src/Databases/enableAllExperimentalSettings.cpp +++ b/src/Databases/enableAllExperimentalSettings.cpp @@ -71,6 +71,7 @@ void enableAllExperimentalSettings(ContextMutablePtr context) context->setSetting("allow_dynamic_type_in_join_keys", 1); context->setSetting("allow_experimental_alias_table_engine", 1); context->setSetting("allow_experimental_database_paimon_rest_catalog", 1); + context->setSetting("allow_experimental_database_s3_tables", 1); context->setSetting("allow_experimental_object_storage_queue_hive_partitioning", 1); context->setSetting("allow_experimental_json_lazy_type_hints", 1); context->setSetting("allow_experimental_url_wildcard_from_index_pages", 1); diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index b671cb447531..3b3226a9bcbb 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -8,6 +8,8 @@ #include #include +#include + #include #include @@ -264,7 +266,11 @@ void URI::validateKey(const String & key, const Poco::URI & uri) } } +<<<<<<< HEAD std::string expandRegionToAmazonPath(const std::string & region) +======= +std::string resolveS3Endpoint(const std::string & region) +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { Aws::S3::Endpoint::S3EndpointProvider provider; provider.AccessBuiltInParameters().SetStringParameter("Region", Aws::String(region)); diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index a8122aa94f40..ad387a6bf4eb 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -52,7 +52,13 @@ struct URI bool tryInitVirtualHostedStyle(bool is_using_aws_private_link_interface, bool use_strict_pattern); }; +<<<<<<< HEAD std::string expandRegionToAmazonPath(const std::string & region); +======= +/// Resolve the S3 endpoint URL for a given AWS region using the SDK's +/// Smithy endpoint rules (handles all partitions: standard, China, GovCloud, etc.). +std::string resolveS3Endpoint(const std::string & region); +>>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } diff --git a/src/IO/S3/tests/gtest_s3_uri.cpp b/src/IO/S3/tests/gtest_s3_uri.cpp index a429c645fca3..5c0d3debc0c0 100644 --- a/src/IO/S3/tests/gtest_s3_uri.cpp +++ b/src/IO/S3/tests/gtest_s3_uri.cpp @@ -38,4 +38,23 @@ TEST(IOTestS3URI, PathStyleWithKey) ASSERT_EQ(uri_with_no_key_and_with_slash.key, "key/key/key/key"); } +TEST(IOTestS3URI, ResolveS3Endpoint) +{ + using namespace DB; + + ASSERT_EQ(S3::resolveS3Endpoint("us-east-1"), + "https://s3.us-east-1.amazonaws.com"); + ASSERT_EQ(S3::resolveS3Endpoint("eu-west-1"), + "https://s3.eu-west-1.amazonaws.com"); + + auto cn_north = S3::resolveS3Endpoint("cn-north-1"); + ASSERT_TRUE(cn_north.ends_with(".amazonaws.com.cn")) + << "China region should resolve to .amazonaws.com.cn suffix, got: " << cn_north; + ASSERT_TRUE(cn_north.find("cn-north-1") != std::string::npos) + << "Got: " << cn_north; + + ASSERT_EQ(S3::resolveS3Endpoint("us-gov-west-1"), + "https://s3.us-gov-west-1.amazonaws.com"); +} + #endif From 4cfdc48c04ba8bb56dce49d89e78b3b3fc094b31 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:49:45 +0200 Subject: [PATCH 2/4] Resolve conflicts in cherry-pick of #2184 antalya-26.8 already carries an independent, newer implementation of the S3 Tables Iceberg REST catalog (S3TablesCatalog, AWSV4Signer, DatabaseDataLakeCatalogType::S3_TABLES, S3 endpoint resolution as DB::S3::expandRegionToAmazonPath), so the S3-Tables-specific halves of the conflicts were resolved in favour of the base branch, and the catalog profile events (PR #1868) were translated onto the base branch shapes: - ProfileEvents.cpp: appended only the rows the source PR adds (the ObjectStorageCluster rows carried by 'theirs' are not in the PR diff). - RestCatalog.cpp/.h: kept the base CatalogState-based createReadBuffer / sendRequest / getAuthHeaders signatures and added the event increments and timers around them. - GlueCatalog.cpp: events added around the base call sites; existsTable now delegates to tryGetTableMetadata on the base branch, where the GetTable event is already counted. - ICatalog.cpp / S3TablesCatalog.* / AWSV4Signer.cpp / SettingsEnums.*: base branch already implements the same behaviour. - ICatalog.h: kept hasDataLakeSpecificProperties() declared, because antalya-26.8 defines TableMetadata::hasDataLakeSpecificProperties(). - gtest_s3_uri.cpp: renamed the source PR's resolveS3Endpoint() calls to the base branch name expandRegionToAmazonPath() (same implementation). Source-PR: #2184 (https://github.com/Altinity/ClickHouse/pull/2184) --- src/Common/ProfileEvents.cpp | 7 - src/Core/SettingsEnums.cpp | 4 - src/Core/SettingsEnums.h | 3 - src/Databases/DataLake/AWSV4Signer.cpp | 22 --- src/Databases/DataLake/DatabaseDataLake.cpp | 15 -- src/Databases/DataLake/GlueCatalog.cpp | 19 +-- src/Databases/DataLake/ICatalog.cpp | 16 -- src/Databases/DataLake/ICatalog.h | 1 + src/Databases/DataLake/RestCatalog.cpp | 138 +++------------- src/Databases/DataLake/RestCatalog.h | 19 --- src/Databases/DataLake/S3TablesCatalog.cpp | 153 ------------------ src/Databases/DataLake/S3TablesCatalog.h | 28 ---- .../tests/gtest_azure_abfss_parsing.cpp | 5 +- src/IO/S3/URI.cpp | 6 - src/IO/S3/URI.h | 6 - src/IO/S3/tests/gtest_s3_uri.cpp | 8 +- 16 files changed, 33 insertions(+), 417 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 0296ff4ae577..cb6e65cb021d 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1659,7 +1659,6 @@ The server successfully detected this situation and will download merged part fr M(AIRowsProcessed, "Number of rows that received an AI result.", ValueType::Number) \ M(AIRowsSkipped, "Number of rows that received a default value due to quota or error.", ValueType::Number) \ \ -<<<<<<< HEAD M(StatelessWorkerRequested, "Number of stateless workers requested by queries for distributed query execution.", ValueType::Number) \ M(StatelessWorkerProvided, "Number of stateless workers provided to queries for distributed query execution.", ValueType::Number) \ M(StatelessWorkerProvisioningMicroseconds, "Total time queries spent waiting for stateless workers to be provisioned.", ValueType::Microseconds) \ @@ -1719,7 +1718,6 @@ The server successfully detected this situation and will download merged part fr M(StatelessWorkerDiscoveryHeartbeatsRejected, "Number of heartbeats the stateless worker discovery service rejected because the worker had already been evicted.", ValueType::Number) \ M(StatelessWorkerDiscoveryKeeperTransactionRetries, "Number of write transactions the stateless worker discovery service retried because its coordination store (Keeper) state was modified concurrently.", ValueType::Number) \ \ -======= M(DataLakeRestCatalogLoadConfig, "Number of 'load config' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogLoadConfigMicroseconds, "Total time of 'load config' requests to Iceberg REST catalog.", ValueType::Microseconds) \ M(DataLakeRestCatalogGetNamespaces, "Number of 'get namespaces' requests to Iceberg REST catalog.", ValueType::Number) \ @@ -1765,11 +1763,6 @@ The server successfully detected this situation and will download merged part fr M(DataLakeUnityCatalogGetCredentials, "Number of 'get credentials' requests to Iceberg Unity catalog.", ValueType::Number) \ M(DataLakeUnityCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ \ - M(ObjectStorageClusterSentToMatchedReplica, "Number of tasks in ObjectStorageCluster request sent to matched replica.", ValueType::Number) \ - M(ObjectStorageClusterSentToNonMatchedReplica, "Number of tasks in ObjectStorageCluster request sent to non-matched replica.", ValueType::Number) \ - M(ObjectStorageClusterProcessedTasks, "Number of processed tasks in ObjectStorageCluster request.", ValueType::Number) \ - M(ObjectStorageClusterWaitingMicroseconds, "Time of waiting for tasks in ObjectStorageCluster request.", ValueType::Microseconds) \ ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #ifdef APPLY_FOR_EXTERNAL_EVENTS #define APPLY_FOR_EVENTS(M) APPLY_FOR_BUILTIN_EVENTS(M) APPLY_FOR_EXTERNAL_EVENTS(M) diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 09b18a45c21e..c385e92cb493 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -377,13 +377,9 @@ IMPLEMENT_SETTING_ENUM( {"onelake", DatabaseDataLakeCatalogType::ICEBERG_ONELAKE}, {"biglake", DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE}, {"paimon_rest", DatabaseDataLakeCatalogType::PAIMON_REST}, -<<<<<<< HEAD {"horizon", DatabaseDataLakeCatalogType::ICEBERG_HORIZON}, {"s3tables", DatabaseDataLakeCatalogType::S3_TABLES}, {"delta_sharing", DatabaseDataLakeCatalogType::ICEBERG_DELTA_SHARING}}) -======= - {"s3tables", DatabaseDataLakeCatalogType::S3_TABLES}}) ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) IMPLEMENT_SETTING_ENUM( FileCachePolicy, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 9022177313d8..c79482c72bab 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -465,11 +465,8 @@ enum class DatabaseDataLakeCatalogType : uint8_t ICEBERG_BIGLAKE, PAIMON_REST, S3_TABLES, -<<<<<<< HEAD ICEBERG_DELTA_SHARING, ICEBERG_HORIZON, -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) }; DECLARE_SETTING_ENUM(DatabaseDataLakeCatalogType) diff --git a/src/Databases/DataLake/AWSV4Signer.cpp b/src/Databases/DataLake/AWSV4Signer.cpp index 722862df213b..53d099bcc7c5 100644 --- a/src/Databases/DataLake/AWSV4Signer.cpp +++ b/src/Databases/DataLake/AWSV4Signer.cpp @@ -12,13 +12,8 @@ #include #include #include -<<<<<<< HEAD #include -======= - -#include ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include namespace DB @@ -37,7 +32,6 @@ namespace Aws::Http::HttpMethod mapPocoMethodToAws(const String & method) { -<<<<<<< HEAD using Poco::Net::HTTPRequest; static const std::pair supported_methods[] = { @@ -47,18 +41,6 @@ Aws::Http::HttpMethod mapPocoMethodToAws(const String & method) {HTTPRequest::HTTP_DELETE, Aws::Http::HttpMethod::HTTP_DELETE}, {HTTPRequest::HTTP_HEAD, Aws::Http::HttpMethod::HTTP_HEAD}, {HTTPRequest::HTTP_PATCH, Aws::Http::HttpMethod::HTTP_PATCH}, -======= - using Aws::Http::HttpMethod; - using Poco::Net::HTTPRequest; - - static const std::pair supported_methods[] = { - {HTTPRequest::HTTP_GET, HttpMethod::HTTP_GET}, - {HTTPRequest::HTTP_POST, HttpMethod::HTTP_POST}, - {HTTPRequest::HTTP_PUT, HttpMethod::HTTP_PUT}, - {HTTPRequest::HTTP_DELETE, HttpMethod::HTTP_DELETE}, - {HTTPRequest::HTTP_HEAD, HttpMethod::HTTP_HEAD}, - {HTTPRequest::HTTP_PATCH, HttpMethod::HTTP_PATCH}, ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) }; for (const auto & [poco_method, aws_method] : supported_methods) @@ -92,11 +74,7 @@ void signRequestWithAWSV4( if (!payload.empty()) { -<<<<<<< HEAD auto body_stream = Aws::MakeShared("AWSV4Signer"); -======= - auto body_stream = Aws::MakeShared("AWSV4Signer"); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) body_stream->write(payload.data(), static_cast(payload.size())); body_stream->seekg(0); request.AddContentBody(body_stream); diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 49cacb16eafe..80849a38f4c4 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -427,12 +427,8 @@ void DatabaseDataLake::initialize() const url, settings[DatabaseDataLakeSetting::region].value, catalog_parameters, -<<<<<<< HEAD Context::getGlobalContextInstance(), allow_server_credentials_in_user_queries); -======= - Context::getGlobalContextInstance()); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #else throw Exception( ErrorCodes::SUPPORT_IS_DISABLED, @@ -529,11 +525,8 @@ std::shared_ptr DatabaseDataLake::getConfigur case DatabaseDataLakeCatalogType::ICEBERG_REST: case DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE: case DatabaseDataLakeCatalogType::S3_TABLES: -<<<<<<< HEAD case DatabaseDataLakeCatalogType::ICEBERG_DELTA_SHARING: case DatabaseDataLakeCatalogType::ICEBERG_HORIZON: -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { switch (type) { @@ -1670,19 +1663,11 @@ void registerDatabaseDataLake(DatabaseFactory & factory) case DatabaseDataLakeCatalogType::S3_TABLES: { if (!args.create_query.attach -<<<<<<< HEAD - && !args.context->getSettingsRef()[Setting::allow_experimental_database_iceberg]) - { - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "DatabaseDataLake with S3 Tables catalog (Iceberg REST) is beta. " - "To allow its usage, enable setting allow_database_iceberg"); -======= && !args.context->getSettingsRef()[Setting::allow_experimental_database_s3_tables]) { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "DatabaseDataLake with S3 Tables catalog is experimental. " "To allow its usage, enable setting allow_experimental_database_s3_tables"); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } engine_func->name = "Iceberg"; diff --git a/src/Databases/DataLake/GlueCatalog.cpp b/src/Databases/DataLake/GlueCatalog.cpp index 37f7f7c49c56..859a73c0fc0a 100644 --- a/src/Databases/DataLake/GlueCatalog.cpp +++ b/src/Databases/DataLake/GlueCatalog.cpp @@ -357,19 +357,8 @@ CatalogTables GlueCatalog::listTablesInNamespaceDirect(const std::string & names bool GlueCatalog::existsTable(const std::string & database_name, const std::string & table_name) const { -<<<<<<< HEAD TableMetadata metadata; return tryGetTableMetadata(database_name, table_name, metadata); -======= - Aws::Glue::Model::GetTableRequest request; - request.SetDatabaseName(database_name); - request.SetName(table_name); - - ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogGetTable); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogGetTableMicroseconds); - auto outcome = glue_client->GetTable(request); - return outcome.IsSuccess(); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } bool GlueCatalog::tryGetTableMetadata( @@ -682,7 +671,8 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons db_input.SetName(namespace_name); create_request.SetDatabaseInput(db_input); -<<<<<<< HEAD + ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); auto outcome = glue_client->CreateDatabase(create_request); if (!outcome.IsSuccess() && outcome.GetError().GetErrorType() != Aws::Glue::GlueErrors::ALREADY_EXISTS) { @@ -691,11 +681,6 @@ void GlueCatalog::createNamespaceIfNotExists(const String & namespace_name, cons "Exception calling CreateDatabase for namespace {}: {}", namespace_name, outcome.GetError().GetMessage()); } -======= - ProfileEvents::increment(ProfileEvents::DataLakeGlueCatalogCreateDatabase); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeGlueCatalogCreateDatabaseMicroseconds); - glue_client->CreateDatabase(create_request); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } void GlueCatalog::createTable(const String & namespace_name, const String & table_name, const String & new_metadata_path, Poco::JSON::Object::Ptr /*metadata_content*/) const diff --git a/src/Databases/DataLake/ICatalog.cpp b/src/Databases/DataLake/ICatalog.cpp index f989b42c795c..e51c96377cdf 100644 --- a/src/Databases/DataLake/ICatalog.cpp +++ b/src/Databases/DataLake/ICatalog.cpp @@ -113,7 +113,6 @@ void TableMetadata::setLocation(const std::string & location_) auto pos_to_bucket = pos + std::strlen("://"); auto path_separator_offset = location_.substr(pos_to_bucket).find('/'); -<<<<<<< HEAD size_t pos_to_path = 0; if (path_separator_offset == std::string::npos) { @@ -126,21 +125,6 @@ void TableMetadata::setLocation(const std::string & location_) throw DB::Exception( DB::ErrorCodes::NOT_IMPLEMENTED, "Location without a path is only supported for the s3 scheme: {}", location_); -======= - if (pos_to_path == std::string::npos) - { - /// An empty path is allowed for AWS S3 Tables: the table location is just `s3://`. - if (storage_type_str == "s3://") - { - location_without_path = location_; - path.clear(); - bucket = location_.substr(pos_to_bucket); - return; - } - - throw DB::Exception(DB::ErrorCodes::NOT_IMPLEMENTED, "Unexpected location format: {}", location_); - } ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) pos_to_path = location_.size(); location_without_path = location_; diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index 57a2adb713b5..e79ebf686f68 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -64,6 +64,7 @@ class TableMetadata bool hasLocation() const; bool hasSchema() const; bool hasStorageCredentials() const; + bool hasDataLakeSpecificProperties() const; void setLocation(const std::string & location_); std::string getLocation() const; diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 562246f9026b..884033092959 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -78,23 +78,10 @@ namespace DB::FailPoints namespace ProfileEvents { -<<<<<<< HEAD extern const Event OneLakeAccessTokenRequests; extern const Event OneLakeAccessTokenRequestFailures; extern const Event OneLakeAccessTokenRequestMicroseconds; extern const Event OneLakeAccessTokenExpirations; -} - -namespace DB::DatabaseDataLakeSetting -{ - extern const DatabaseDataLakeSettingsString catalog_credential; - extern const DatabaseDataLakeSettingsString auth_header; - extern const DatabaseDataLakeSettingsString onelake_tenant_id; - extern const DatabaseDataLakeSettingsString onelake_bearer_token; - extern const DatabaseDataLakeSettingsString onelake_refresh_token; - extern const DatabaseDataLakeSettingsString onelake_client_id; - extern const DatabaseDataLakeSettingsString onelake_client_secret; -======= extern const Event DataLakeRestCatalogLoadConfig; extern const Event DataLakeRestCatalogLoadConfigMicroseconds; extern const Event DataLakeRestCatalogGetNamespaces; @@ -113,7 +100,17 @@ namespace DB::DatabaseDataLakeSetting extern const Event DataLakeRestCatalogUpdateTableMicroseconds; extern const Event DataLakeRestCatalogDropTable; extern const Event DataLakeRestCatalogDropTableMicroseconds; ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) +} + +namespace DB::DatabaseDataLakeSetting +{ + extern const DatabaseDataLakeSettingsString catalog_credential; + extern const DatabaseDataLakeSettingsString auth_header; + extern const DatabaseDataLakeSettingsString onelake_tenant_id; + extern const DatabaseDataLakeSettingsString onelake_bearer_token; + extern const DatabaseDataLakeSettingsString onelake_refresh_token; + extern const DatabaseDataLakeSettingsString onelake_client_id; + extern const DatabaseDataLakeSettingsString onelake_client_secret; } namespace DataLake @@ -270,17 +267,13 @@ RestCatalog::RestCatalog( RestCatalog::Config RestCatalog::loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers) { Poco::URI::QueryParameters params = {{"warehouse", warehouse}}; -<<<<<<< HEAD - auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */ {}, auth_headers); -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::string json_str; { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogLoadConfig); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogLoadConfigMicroseconds); - auto buf = createReadBuffer(CONFIG_ENDPOINT, params); + auto buf = createReadBuffer(catalog_state, CONFIG_ENDPOINT, params, /* headers */ {}, auth_headers); readJSONObjectPossiblyInvalid(json_str, *buf); } @@ -326,16 +319,7 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } -<<<<<<< HEAD DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const -======= -DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( - bool update_token, - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/) const ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -1037,16 +1021,7 @@ BigLakeCatalog::BigLakeCatalog( state.set(std::make_unique(std::move(initial_state))); } -<<<<<<< HEAD DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const -======= -DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( - bool update_token, - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/) const ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -1233,11 +1208,7 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token) { -<<<<<<< HEAD auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token); -======= - auto result_headers = getAuthHeaders(update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1433,14 +1404,10 @@ RestCatalog::Namespaces RestCatalog::listChildNamespaces(const std::string & bas if (!page_token.empty()) params.push_back({"pageToken", page_token}); -<<<<<<< HEAD - auto buf = createReadBuffer( - *state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params, /* headers */ {}, /* auth_headers */ std::nullopt); -======= ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetNamespaces); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetNamespacesMicroseconds); - auto buf = createReadBuffer(config.prefix / NAMESPACES_ENDPOINT, params); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + auto buf = createReadBuffer( + *state_snapshot, state_snapshot->config.prefix / NAMESPACES_ENDPOINT, params, /* headers */ {}, /* auth_headers */ std::nullopt); String next_page_token; auto page_namespaces = parseNamespaces(*buf, base_namespace, next_page_token); LOG_DEBUG( @@ -1590,14 +1557,10 @@ DB::Names RestCatalog::listTablesInNamespace(const std::string & base_namespace, if (!page_token.empty()) params.push_back({"pageToken", page_token}); -<<<<<<< HEAD - auto buf = createReadBuffer( - *state_snapshot, state_snapshot->config.prefix / endpoint, params, /* headers */ {}, /* auth_headers */ std::nullopt); -======= ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTables); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTablesMicroseconds); - auto buf = createReadBuffer(config.prefix / endpoint, params); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + auto buf = createReadBuffer( + *state_snapshot, state_snapshot->config.prefix / endpoint, params, /* headers */ {}, /* auth_headers */ std::nullopt); /// Pass through the remaining limit so that single-page short-circuiting still works /// when the caller is in `empty()` (limit=1) and the first page already contains a row. @@ -1742,23 +1705,13 @@ bool RestCatalog::getTableMetadataImpl( const auto state_snapshot = state.get(); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; -<<<<<<< HEAD - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); - - if (buf->eof()) - { - LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); - return false; - } - -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) String json_str; { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetTableMetadata); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetTableMetadataMicroseconds); - auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); if (buf->eof()) { @@ -1842,15 +1795,12 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & request_body->stringify(oss); const std::string body_str = DB::removeEscapedSlashes(oss.str()); -<<<<<<< HEAD LOG_TEST(log, "REST catalog {} {} body ({} bytes): {}", method, endpoint, body_str.size(), body_str); DB::HTTPHeaderEntries headers = getAuthHeaders(catalog_state, /* update_token = */ true); headers.emplace_back("Content-Type", "application/json"); headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) const auto & context = getContext(); DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; @@ -1865,14 +1815,6 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & /// enable_url_encoding=false to allow using tables with encoded sequences in names like 'foo%2Fbar' Poco::URI url(endpoint, /* enable_url_encoding */ false); -<<<<<<< HEAD -======= - DB::HTTPHeaderEntries extra_headers; - extra_headers.emplace_back("Content-Type", "application/json"); - - DB::HTTPHeaderEntries headers = getAuthHeaders(/* update_token = */ true, method, url, extra_headers, body_str); - headers.emplace_back("Content-Type", "application/json"); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withMethod(method) @@ -1929,13 +1871,9 @@ void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, cons try { -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); -======= ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateNamespace); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateNamespaceMicroseconds); - sendRequest(endpoint, request_body); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); } catch (const DB::HTTPException & e) { @@ -1979,13 +1917,9 @@ void RestCatalog::createTable(const String & namespace_name, const String & tabl try { -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); -======= ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogCreateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogCreateTableMicroseconds); - sendRequest(endpoint, request_body); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); } catch (const DB::HTTPException & ex) { @@ -2052,13 +1986,9 @@ bool RestCatalog::updateMetadata(const String & namespace_name, const String & t try { -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); -======= ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUpdateTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogUpdateTableMicroseconds); - sendRequest(endpoint, request_body); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_POST, /* ignore_result */ false); } catch (const DB::HTTPException & ex) { @@ -2144,25 +2074,15 @@ bool RestCatalog::updateSchema( void RestCatalog::dropTable(const String & namespace_name, const String & table_name, bool /*delete_data*/) const { -<<<<<<< HEAD const auto state_snapshot = state.get(); const std::string endpoint = fmt::format("{}/namespaces/{}/tables/{}?purgeRequested=False", base_url, namespace_name, table_name); -======= - const std::string endpoint - = (base_url / config.prefix / NAMESPACES_ENDPOINT / encodeNamespaceForURI(namespace_name) / "tables" / table_name).generic_string() - + "?purgeRequested=False"; ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) Poco::JSON::Object::Ptr request_body = nullptr; try { -<<<<<<< HEAD - sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); -======= ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); } catch (const DB::HTTPException & ex) { @@ -2250,23 +2170,13 @@ ICatalog::CredentialsRefreshCallback RestCatalog::getCredentialsConfigurationCal const auto & table = storage_id.getTableName(); auto [namespace_name, table_name] = DataLake::parseTableName(table); const std::string endpoint = std::filesystem::path(NAMESPACES_ENDPOINT) / encodeNamespaceForURI(namespace_name) / "tables" / table_name; -<<<<<<< HEAD - auto buf = createReadBuffer(*state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); - - if (buf->eof()) - { - LOG_DEBUG(log, "Table doesn't exist (endpoint: {})", endpoint); - return nullptr; - } - -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) String json_str; { ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogGetCredentials); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogGetCredentialsMicroseconds); - auto buf = createReadBuffer(config.prefix / endpoint, /* params */{}, headers); + auto buf = createReadBuffer( + *state_snapshot, state_snapshot->config.prefix / endpoint, /* params */ {}, headers, /* auth_headers */ std::nullopt); if (buf->eof()) { diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index ee473cc6d0cf..d826bbebc8e8 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -205,19 +205,9 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, TableMetadata & result) const; -<<<<<<< HEAD /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); virtual DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const; -======= - Config loadConfig(); - virtual DB::HTTPHeaderEntries getAuthHeaders( - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}) const; ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -336,16 +326,7 @@ class BigLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE; } -<<<<<<< HEAD DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const override; -======= - DB::HTTPHeaderEntries getAuthHeaders( - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}) const override; ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index a86467bdaadf..bbf1365250b5 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -4,18 +4,10 @@ #include #include -<<<<<<< HEAD -======= -#include ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include #include -<<<<<<< HEAD -======= -#include ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include #include @@ -25,53 +17,35 @@ #include #include #include -<<<<<<< HEAD #include #include #include #include #include -======= -#include -#include -#include ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include #include #include -<<<<<<< HEAD #include -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) namespace DB::ErrorCodes { extern const int BAD_ARGUMENTS; extern const int DATALAKE_DATABASE_ERROR; -<<<<<<< HEAD extern const int SUPPORT_IS_DISABLED; -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } namespace DB::Setting { -<<<<<<< HEAD extern const SettingsUInt64 s3_max_connections; -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) extern const SettingsUInt64 s3_max_redirects; extern const SettingsUInt64 s3_retry_attempts; extern const SettingsBool s3_slow_all_threads_after_network_error; extern const SettingsBool enable_s3_requests_logging; -<<<<<<< HEAD extern const SettingsUInt64 s3_connect_timeout_ms; extern const SettingsUInt64 s3_request_timeout_ms; -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } namespace DB::ServerSetting @@ -80,15 +54,6 @@ namespace DB::ServerSetting extern const ServerSettingsUInt64 s3_retry_attempts; } -<<<<<<< HEAD -======= -namespace ProfileEvents -{ - extern const Event DataLakeRestCatalogDropTable; - extern const Event DataLakeRestCatalogDropTableMicroseconds; -} - ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) namespace DataLake { @@ -97,19 +62,12 @@ S3TablesCatalog::S3TablesCatalog( const String & base_url_, const String & region_, const CatalogSettings & catalog_settings_, -<<<<<<< HEAD DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_) : RestCatalog(warehouse_, base_url_, "", "", false, context_) , region(region_) , storage_endpoint(catalog_settings_.storage_endpoint) , signing_service("s3tables") -======= - DB::ContextPtr context_) - : RestCatalog(warehouse_, base_url_, "", "", false, context_) - , region(region_) - , storage_endpoint(catalog_settings_.storage_endpoint) ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { if (region.empty()) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "S3 Tables catalog requires non-empty `region` setting"); @@ -119,15 +77,12 @@ S3TablesCatalog::S3TablesCatalog( creds_config.role_arn = catalog_settings_.aws_role_arn; creds_config.role_session_name = catalog_settings_.aws_role_session_name; -<<<<<<< HEAD /// An S3 Tables catalog is created by user SQL, so it must not reuse the server's own credentials unless /// that was allowed at CREATE time. The cached catalog holds the global context, whose live setting never /// reflects the creating session, so pass the value captured then instead. creds_config.forbid_implicit_credentials = getContext()->shouldRestrictUserQueryS3Credentials(allow_server_credentials_in_user_queries_); -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) const auto & server_settings = getContext()->getGlobalContext()->getServerSettings(); const DB::Settings & global_settings = getContext()->getGlobalContext()->getSettingsRef(); @@ -165,7 +120,6 @@ S3TablesCatalog::S3TablesCatalog( Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, /* urlEscapePath = */ false); -<<<<<<< HEAD /// The signer is fully initialised above, so the virtual `createReadBuffer` reached by /// `loadConfig` dispatches to this class's SigV4 implementation. CatalogState initial_state; @@ -187,23 +141,6 @@ S3TablesCatalog::S3TablesCatalog( CatalogTables S3TablesCatalog::getTables() const { auto namespaces = listChildNamespaces(""); -======= - config = loadConfig(); - - if (config.prefix.empty()) - { - String encoded_warehouse; - Poco::URI::encode(warehouse_, "", encoded_warehouse); - config.prefix = encoded_warehouse; - } -} - -/// S3 Tables only supports a single level of namespaces (no nesting), -/// so we use flat getNamespaces() instead of the base class's getNamespacesRecursive(). -DB::Names S3TablesCatalog::getTables() const -{ - auto namespaces = getNamespaces(""); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) auto & pool = getContext()->getIcebergCatalogThreadpool(); DB::ThreadPoolCallbackRunnerLocal runner(pool, DB::ThreadName::DATALAKE_REST_CATALOG); @@ -215,17 +152,12 @@ DB::Names S3TablesCatalog::getTables() const runner.enqueueAndKeepTrack( [&, ns] { -<<<<<<< HEAD auto tables_in_ns = listTablesInNamespace(ns); -======= - auto tables_in_ns = RestCatalog::getTables(ns); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::lock_guard lock(mutex); std::move(tables_in_ns.begin(), tables_in_ns.end(), std::back_inserter(tables)); }); } runner.waitForAllToFinishAndRethrowFirstError(); -<<<<<<< HEAD /// A REST catalog is Iceberg-only, so every listed table is readable. CatalogTables result; @@ -233,9 +165,6 @@ DB::Names S3TablesCatalog::getTables() const for (auto & name : tables) result.push_back(CatalogTable{.name = std::move(name)}); return result; -======= - return tables; ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } bool S3TablesCatalog::tryGetTableMetadata( @@ -246,7 +175,6 @@ bool S3TablesCatalog::tryGetTableMetadata( if (!RestCatalog::tryGetTableMetadata(namespace_name, table_name, result)) return false; -<<<<<<< HEAD /// For S3 Tables the catalog and the underlying data live in AWS S3 under the same /// AWS principal, so endpoint/metadata-location normalization and fallback IAM /// credential injection must always run - even when the engine was created with @@ -263,15 +191,6 @@ bool S3TablesCatalog::tryGetTableMetadata( if (result.hasStorageCredentials()) { auto creds = std::dynamic_pointer_cast(result.getStorageCredentials()); -======= - if (!result.requiresCredentials()) - return true; - - bool need_credentials = true; - if (const auto storage_credentials = result.getStorageCredentials()) - { - auto creds = std::dynamic_pointer_cast(storage_credentials); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) if (creds && !creds->isEmpty()) need_credentials = false; } @@ -280,16 +199,7 @@ bool S3TablesCatalog::tryGetTableMetadata( { LOG_DEBUG(log, "S3 Tables: no vended credentials for {}.{}, injecting catalog IAM credentials", namespace_name, table_name); auto aws_creds = credentials_provider->GetAWSCredentials(); -<<<<<<< HEAD result.withStorageCredentials(); -======= - if (aws_creds.GetAWSAccessKeyId().empty() || aws_creds.GetAWSSecretKey().empty()) - throw DB::Exception( - DB::ErrorCodes::BAD_ARGUMENTS, - "S3 Tables: catalog IAM credentials are empty for {}.{}, " - "check AWS credentials configuration", - namespace_name, table_name); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) result.setStorageCredentials(std::make_shared( aws_creds.GetAWSAccessKeyId(), aws_creds.GetAWSSecretKey(), aws_creds.GetSessionToken())); } @@ -297,17 +207,12 @@ bool S3TablesCatalog::tryGetTableMetadata( if (result.getEndpoint().empty()) { String endpoint = storage_endpoint.empty() -<<<<<<< HEAD ? DB::S3::expandRegionToAmazonPath(region) -======= - ? DB::S3::resolveS3Endpoint(region) ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) : storage_endpoint; LOG_DEBUG(log, "S3 Tables: no endpoint for {}.{}, injecting: {}", namespace_name, table_name, endpoint); result.setEndpoint(endpoint); } -<<<<<<< HEAD if (auto props = result.getDataLakeSpecificProperties(); props && !props->iceberg_metadata_file_location.empty()) { @@ -338,63 +243,24 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta const auto state_snapshot = state.get(); const std::string endpoint = (base_url / state_snapshot->config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() -======= - return true; -} - -ICatalog::CredentialsRefreshCallback S3TablesCatalog::getCredentialsConfigurationCallback(const DB::StorageID & storage_id) -{ - auto base_cb = RestCatalog::getCredentialsConfigurationCallback(storage_id); - return [this, base_callback = std::move(base_cb)] () -> std::shared_ptr - { - if (base_callback) - { - if (auto creds = (*base_callback)()) - { - auto s3_creds = std::dynamic_pointer_cast(creds); - if (s3_creds && !s3_creds->isEmpty()) - return creds; - } - LOG_DEBUG(log, "S3 Tables: vended credentials unavailable on refresh, falling back to catalog IAM credentials"); - } - - return resolveS3TablesRefreshCredentials(std::nullopt, *credentials_provider); - }; -} - -void S3TablesCatalog::dropTable(const String & namespace_name, const String & table_name) const -{ - const std::string endpoint - = (base_url / config.prefix / "namespaces" / namespace_name / "tables" / table_name).string() ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) + "?purgeRequested=True"; Poco::JSON::Object::Ptr request_body = nullptr; try { -<<<<<<< HEAD sendRequest(*state_snapshot, endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); LOG_INFO(log, "S3 Tables: dropped table {}.{} (purgeRequested=True)", namespace_name, table_name); -======= - ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogDropTable); - auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogDropTableMicroseconds); - sendRequest(endpoint, request_body, Poco::Net::HTTPRequest::HTTP_DELETE, true); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } catch (const DB::HTTPException & ex) { if (ex.getHTTPStatus() == Poco::Net::HTTPResponse::HTTP_NOT_FOUND) -<<<<<<< HEAD // 404 is returned by the API when the table does not exist -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) LOG_DEBUG(log, "S3 Tables: table {}.{} already does not exist (404 on purge-delete)", namespace_name, table_name); else throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "Failed to drop table {}", ex.displayText()); } } -<<<<<<< HEAD namespace { @@ -418,22 +284,6 @@ DB::HTTPHeaderEntries extractSigV4AuthHeaders(DB::HTTPHeaderEntries && all_signe for (auto & h : all_signed) { if (isSigV4AuthHeader(h.name)) -======= -DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( - bool /*update_token*/, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body) const -{ - DB::HTTPHeaderEntries all_signed; - signRequestWithAWSV4(method, url, extra_headers, body, *signer, region, "s3tables", all_signed); - - DB::HTTPHeaderEntries auth_headers; - for (auto & h : all_signed) - { - if (h.name == "authorization" || h.name.starts_with("x-amz-")) ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) auth_headers.push_back(std::move(h)); } return auth_headers; @@ -441,7 +291,6 @@ DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( } -<<<<<<< HEAD DB::ReadWriteBufferFromHTTPPtr S3TablesCatalog::createReadBuffer( const CatalogState & /* catalog_state */, const std::string & endpoint, @@ -530,6 +379,4 @@ void S3TablesCatalog::sendRequest( } -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #endif diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index a2befada02c2..65ca66500ed1 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -7,11 +7,8 @@ #include #include -<<<<<<< HEAD #include -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) #include #include @@ -34,7 +31,6 @@ class S3TablesCatalog final : public RestCatalog const String & base_url_, const String & region_, const DataLake::CatalogSettings & catalog_settings_, -<<<<<<< HEAD DB::ContextPtr context_, bool allow_server_credentials_in_user_queries_); @@ -43,20 +39,12 @@ class S3TablesCatalog final : public RestCatalog CatalogTables getTables() const override; bool managesTableLocation() const override { return true; } -======= - DB::ContextPtr context_); - - DB::DatabaseDataLakeCatalogType getCatalogType() const override { return DB::DatabaseDataLakeCatalogType::S3_TABLES; } - - DB::Names getTables() const override; ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) bool tryGetTableMetadata( const std::string & namespace_name, const std::string & table_name, TableMetadata & result) const override; -<<<<<<< HEAD void dropTable(const String & namespace_name, const String & table_name, bool delete_data) const override; protected: @@ -77,27 +65,11 @@ class S3TablesCatalog final : public RestCatalog Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const override; -======= - void dropTable(const String & namespace_name, const String & table_name) const override; - - ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & storage_id) override; - -protected: - DB::HTTPHeaderEntries getAuthHeaders( - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}) const override; ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) private: const String region; const String storage_endpoint; -<<<<<<< HEAD const String signing_service; -======= ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) std::shared_ptr credentials_provider; std::unique_ptr signer; }; diff --git a/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp b/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp index a8b4703ed8ac..58f4925b1bb8 100644 --- a/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp +++ b/src/Databases/DataLake/tests/gtest_azure_abfss_parsing.cpp @@ -240,7 +240,6 @@ TEST_F(AzureAbfssParsingTest, TableMetadataGetLocationWithEndpointVirtualHostedD EXPECT_EQ(location, "https://my.dotted.bucket.s3.mycompany.com/path/to/table/"); } -<<<<<<< HEAD TEST_F(AzureAbfssParsingTest, TableMetadataAbfssEndpointAlreadyContainsContainerDefault) { TableMetadata metadata; @@ -283,7 +282,8 @@ TEST_F(AzureAbfssParsingTest, TableMetadataS3EndpointAlreadyEndsWithBucketForceA metadata.setEndpoint("http://minio:9000/warehouse-rest"); EXPECT_EQ(metadata.getLocation(), "http://minio:9000/warehouse-rest/warehouse-rest/data/testns/testtable/"); -======= +} + TEST_F(AzureAbfssParsingTest, TableMetadataGetMetadataLocationS3TablesWithAwsEndpoint) { TableMetadata metadata; @@ -309,7 +309,6 @@ TEST_F(AzureAbfssParsingTest, TableMetadataGetMetadataLocationS3TablesEmptyPathW const std::string metadata_file = "s3://bucket/metadata/v1.metadata.json"; EXPECT_EQ(metadata.getMetadataLocation(metadata_file), "metadata/v1.metadata.json"); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } } diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index 3b3226a9bcbb..b671cb447531 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -8,8 +8,6 @@ #include #include -#include - #include #include @@ -266,11 +264,7 @@ void URI::validateKey(const String & key, const Poco::URI & uri) } } -<<<<<<< HEAD std::string expandRegionToAmazonPath(const std::string & region) -======= -std::string resolveS3Endpoint(const std::string & region) ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) { Aws::S3::Endpoint::S3EndpointProvider provider; provider.AccessBuiltInParameters().SetStringParameter("Region", Aws::String(region)); diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h index ad387a6bf4eb..a8122aa94f40 100644 --- a/src/IO/S3/URI.h +++ b/src/IO/S3/URI.h @@ -52,13 +52,7 @@ struct URI bool tryInitVirtualHostedStyle(bool is_using_aws_private_link_interface, bool use_strict_pattern); }; -<<<<<<< HEAD std::string expandRegionToAmazonPath(const std::string & region); -======= -/// Resolve the S3 endpoint URL for a given AWS region using the SDK's -/// Smithy endpoint rules (handles all partitions: standard, China, GovCloud, etc.). -std::string resolveS3Endpoint(const std::string & region); ->>>>>>> 55471e34c35 (Merge pull request #2184 from Altinity/feature/antalya-26.6/auto-grp-pr-1808) } diff --git a/src/IO/S3/tests/gtest_s3_uri.cpp b/src/IO/S3/tests/gtest_s3_uri.cpp index 5c0d3debc0c0..5fce6f693d25 100644 --- a/src/IO/S3/tests/gtest_s3_uri.cpp +++ b/src/IO/S3/tests/gtest_s3_uri.cpp @@ -42,18 +42,18 @@ TEST(IOTestS3URI, ResolveS3Endpoint) { using namespace DB; - ASSERT_EQ(S3::resolveS3Endpoint("us-east-1"), + ASSERT_EQ(S3::expandRegionToAmazonPath("us-east-1"), "https://s3.us-east-1.amazonaws.com"); - ASSERT_EQ(S3::resolveS3Endpoint("eu-west-1"), + ASSERT_EQ(S3::expandRegionToAmazonPath("eu-west-1"), "https://s3.eu-west-1.amazonaws.com"); - auto cn_north = S3::resolveS3Endpoint("cn-north-1"); + auto cn_north = S3::expandRegionToAmazonPath("cn-north-1"); ASSERT_TRUE(cn_north.ends_with(".amazonaws.com.cn")) << "China region should resolve to .amazonaws.com.cn suffix, got: " << cn_north; ASSERT_TRUE(cn_north.find("cn-north-1") != std::string::npos) << "Got: " << cn_north; - ASSERT_EQ(S3::resolveS3Endpoint("us-gov-west-1"), + ASSERT_EQ(S3::expandRegionToAmazonPath("us-gov-west-1"), "https://s3.us-gov-west-1.amazonaws.com"); } From f8007090fea398049f874e26904601dffc7b0e5d Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:05:26 +0200 Subject: [PATCH 3/4] Cherry-pick of https://github.com/Altinity/ClickHouse/pull/2222 with unresolved conflict markers (resolution in next commit) --- Original cherry-pick message follows: Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events Antalya-26.6: Datalake catalog auth token profile events # Conflicts: # src/Databases/DataLake/RestCatalog.cpp # src/Databases/DataLake/RestCatalog.h # src/Databases/DataLake/S3TablesCatalog.cpp # src/Databases/DataLake/S3TablesCatalog.h # tests/integration/test_database_iceberg_lakekeeper_catalog/test.py --- src/Common/ProfileEvents.cpp | 5 + src/Databases/DataLake/RestCatalog.cpp | 142 +++++++++- src/Databases/DataLake/RestCatalog.h | 21 ++ src/Databases/DataLake/S3TablesCatalog.cpp | 10 + src/Databases/DataLake/S3TablesCatalog.h | 10 + ...ker_compose_iceberg_lakekeeper_catalog.yml | 24 ++ .../test.py | 248 ++++++++++++++++++ 7 files changed, 451 insertions(+), 9 deletions(-) diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index cb6e65cb021d..ed50cfd324c2 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1728,6 +1728,10 @@ The server successfully detected this situation and will download merged part fr M(DataLakeRestCatalogGetTableMetadataMicroseconds, "Total time of 'get table metadata' requests to Iceberg REST catalog.", ValueType::Microseconds) \ M(DataLakeRestCatalogGetCredentials, "Number of 'get credentials' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogAuthTokenCachedValid, "Number of requests to Iceberg REST catalog that reused a cached access token and did not fetch a new one.", ValueType::Number) \ + M(DataLakeRestCatalogAuthTokenRetrieve, "Number of new access tokens fetched for Iceberg REST catalog (OAuth client-credentials or GCP metadata/ADC).", ValueType::Number) \ + M(DataLakeRestCatalogAuthTokenRefreshedMicroseconds, "Total time spent fetching access tokens for Iceberg REST catalog.", ValueType::Microseconds) \ + M(DataLakeRestCatalogUnauthorized, "Number of Iceberg REST catalog HTTP requests retried with a new access token after HTTP 401 or 403.", ValueType::Number) \ M(DataLakeRestCatalogCreateNamespace, "Number of 'create namespace' requests to Iceberg REST catalog.", ValueType::Number) \ M(DataLakeRestCatalogCreateNamespaceMicroseconds, "Total time of 'create namespace' requests to Iceberg REST catalog.", ValueType::Microseconds) \ M(DataLakeRestCatalogCreateTable, "Number of 'create table' requests to Iceberg REST catalog.", ValueType::Number) \ @@ -1764,6 +1768,7 @@ The server successfully detected this situation and will download merged part fr M(DataLakeUnityCatalogGetCredentialsMicroseconds, "Total time of 'get credentials' requests to Iceberg Unity catalog.", ValueType::Microseconds) \ \ + #ifdef APPLY_FOR_EXTERNAL_EVENTS #define APPLY_FOR_EVENTS(M) APPLY_FOR_BUILTIN_EVENTS(M) APPLY_FOR_EXTERNAL_EVENTS(M) #else diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index 884033092959..d85a983a7d95 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -92,6 +92,10 @@ namespace ProfileEvents extern const Event DataLakeRestCatalogGetTableMetadataMicroseconds; extern const Event DataLakeRestCatalogGetCredentials; extern const Event DataLakeRestCatalogGetCredentialsMicroseconds; + extern const Event DataLakeRestCatalogAuthTokenCachedValid; + extern const Event DataLakeRestCatalogAuthTokenRetrieve; + extern const Event DataLakeRestCatalogAuthTokenRefreshedMicroseconds; + extern const Event DataLakeRestCatalogUnauthorized; extern const Event DataLakeRestCatalogCreateNamespace; extern const Event DataLakeRestCatalogCreateNamespaceMicroseconds; extern const Event DataLakeRestCatalogCreateTable; @@ -319,13 +323,26 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } +<<<<<<< HEAD DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const +======= +DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( + bool update_token, + const String & /*method*/, + const Poco::URI & /*url*/, + const DB::HTTPHeaderEntries & /*extra_headers*/, + const String & /*body*/, + bool * used_cached_oauth_token) const +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { throw DB::Exception(DB::ErrorCodes::FAULT_INJECTED, "Injecting fault when checking database"); }); + if (used_cached_oauth_token) + *used_cached_oauth_token = false; + /// Option 1: user specified auth header manually. /// Header has format: 'Authorization: '. if (catalog_state.auth_header.has_value()) @@ -343,11 +360,15 @@ DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const CatalogState & catalog_s /// request fails with 401/403 and is retried with `update_token = true`, fetching /// a token with the snapshot's credentials. auto current = access_token.get(); - if (!current || update_token) + if (!current || update_token || current->isExpired()) { access_token.set(std::make_unique(retrieveAccessToken(catalog_state.client_id, catalog_state.client_secret))); current = access_token.get(); } + else if (used_cached_oauth_token) + { + *used_cached_oauth_token = true; + } DB::HTTPHeaderEntries headers; headers.emplace_back("Authorization", "Bearer " + current->token); @@ -767,6 +788,9 @@ namespace AccessToken RestCatalog::retrieveAccessToken(const std::string & client_id, const std::string & client_secret) const { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); + static constexpr auto oauth_tokens_endpoint = "oauth/tokens"; /// TODO: @@ -1021,7 +1045,17 @@ BigLakeCatalog::BigLakeCatalog( state.set(std::make_unique(std::move(initial_state))); } +<<<<<<< HEAD DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const +======= +DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( + bool update_token, + const String & method, + const Poco::URI & url, + const DB::HTTPHeaderEntries & extra_headers, + const String & body, + bool * used_cached_oauth_token) const +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -1029,12 +1063,19 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const CatalogState & catalo /// https://developers.google.com/identity/protocols/oauth2 if (!google_project_id.empty() || !google_adc_client_id.empty()) { + if (used_cached_oauth_token) + *used_cached_oauth_token = false; + auto current = access_token.get(); if (!current || update_token || current->isExpired()) { access_token.set(std::make_unique(retrieveGoogleCloudAccessToken())); current = access_token.get(); } + else if (used_cached_oauth_token) + { + *used_cached_oauth_token = true; + } DB::HTTPHeaderEntries headers; headers.emplace_back("Authorization", "Bearer " + current->token); @@ -1053,7 +1094,11 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const CatalogState & catalo return headers; } +<<<<<<< HEAD return RestCatalog::getAuthHeaders(catalog_state, update_token); +======= + return RestCatalog::getAuthHeaders(update_token, method, url, extra_headers, body, used_cached_oauth_token); +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1075,7 +1120,24 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() con AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const { +<<<<<<< HEAD const auto & context = getContext(); +======= + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); + auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); + + if (!google_adc_client_id.empty() && !google_adc_client_secret.empty() && !google_adc_refresh_token.empty()) + { + try + { + return retrieveGoogleCloudAccessTokenFromRefreshToken(); + } + catch (const DB::Exception & e) + { + LOG_DEBUG(log, "Failed to use ADC credentials, falling back to metadata service: {}", e.what()); + } + } +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) /// An explicit Application Default Credentials triple is a user-supplied credential, so it is honored. /// Fail closed if it does not work: do not fall back to the server's GCP metadata service, which would @@ -1206,9 +1268,13 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( if (!params.empty()) url.setQueryParameters(params); - auto create_buffer = [&](bool update_token) + auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { +<<<<<<< HEAD auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token); +======= + auto result_headers = getAuthHeaders(update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}, &used_cached_oauth_token); +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1226,7 +1292,11 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( try { - return create_buffer(false); + bool used_cached_oauth_token = false; + auto buf = create_buffer(false, used_cached_oauth_token); + if (used_cached_oauth_token) + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenCachedValid); + return buf; } catch (const DB::HTTPException & e) { @@ -1235,7 +1305,9 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) { - return create_buffer(true); + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + return create_buffer(true, used_cached_oauth_token_on_retry); } throw; } @@ -1815,6 +1887,7 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & /// enable_url_encoding=false to allow using tables with encoded sequences in names like 'foo%2Fbar' Poco::URI url(endpoint, /* enable_url_encoding */ false); +<<<<<<< HEAD auto wb = DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withMethod(method) @@ -1828,12 +1901,63 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & .withOutCallbackFixedContentLength(body_str.size()) .withSkipNotFound(false) .create(credentials); +======= + DB::HTTPHeaderEntries extra_headers; + extra_headers.emplace_back("Content-Type", "application/json"); - String response_str; - if (!ignore_result) - readJSONObjectPossiblyInvalid(response_str, *wb); - else - wb->ignoreAll(); + auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) + { + DB::HTTPHeaderEntries headers = getAuthHeaders(update_token, method, url, extra_headers, body_str, &used_cached_oauth_token); + headers.emplace_back("Content-Type", "application/json"); + return DB::BuilderRWBufferFromHTTP(url) + .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) + .withMethod(method) + .withSettings(context->getReadSettings()) + .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) + .withHostFilter(&context->getRemoteHostFilter()) + .withHeaders(headers) + .withOutCallback(out_stream_callback) + .withSkipNotFound(false) + .create(credentials); + }; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) + + try + { + bool used_cached_oauth_token = false; + auto wb = create_buffer(false, used_cached_oauth_token); + + String response_str; + if (!ignore_result) + readJSONObjectPossiblyInvalid(response_str, *wb); + else + wb->ignoreAll(); + + if (used_cached_oauth_token) + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenCachedValid); + } + catch (const DB::HTTPException & e) + { + const auto status = e.getHTTPStatus(); + if (update_token_if_expired && + (status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_UNAUTHORIZED + || status == Poco::Net::HTTPResponse::HTTPStatus::HTTP_FORBIDDEN)) + { + ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogUnauthorized); + bool used_cached_oauth_token_on_retry = false; + auto wb = create_buffer(true, used_cached_oauth_token_on_retry); + + String response_str; + if (!ignore_result) + readJSONObjectPossiblyInvalid(response_str, *wb); + else + wb->ignoreAll(); + } + else + { + throw; + } + } } void RestCatalog::createNamespaceIfNotExists(const String & namespace_name, const String & location) const diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index d826bbebc8e8..c33c3feb9da3 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -205,9 +205,20 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, TableMetadata & result) const; +<<<<<<< HEAD /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); virtual DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const; +======= + Config loadConfig(); + virtual DB::HTTPHeaderEntries getAuthHeaders( + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}, + bool * used_cached_oauth_token = nullptr) const; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -326,7 +337,17 @@ class BigLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE; } +<<<<<<< HEAD DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const override; +======= + DB::HTTPHeaderEntries getAuthHeaders( + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}, + bool * used_cached_oauth_token = nullptr) const override; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index bbf1365250b5..1b3d9152d2de 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -261,7 +261,17 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } +<<<<<<< HEAD namespace +======= +DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( + bool /*update_token*/, + const String & method, + const Poco::URI & url, + const DB::HTTPHeaderEntries & extra_headers, + const String & body, + bool * /*used_cached_oauth_token*/) const +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { /// `signRequestWithAWSV4` returns the full set of headers that the AWS SDK kept on diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index 65ca66500ed1..d5d93f0f35dc 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -48,6 +48,7 @@ class S3TablesCatalog final : public RestCatalog void dropTable(const String & namespace_name, const String & table_name, bool delete_data) const override; protected: +<<<<<<< HEAD /// Override the network primitives instead of `getAuthHeaders` so the SigV4 signer has /// access to the final URL, method, and request body for canonicalisation. /// `catalog_state` and `auth_headers` are unused here: authentication is derived from the @@ -65,6 +66,15 @@ class S3TablesCatalog final : public RestCatalog Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const override; +======= + DB::HTTPHeaderEntries getAuthHeaders( + bool update_token, + const String & method = {}, + const Poco::URI & url = {}, + const DB::HTTPHeaderEntries & extra_headers = {}, + const String & body = {}, + bool * used_cached_oauth_token = nullptr) const override; +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) private: const String region; diff --git a/tests/integration/compose/docker_compose_iceberg_lakekeeper_catalog.yml b/tests/integration/compose/docker_compose_iceberg_lakekeeper_catalog.yml index 834dfcf071c9..a2f68b606c77 100644 --- a/tests/integration/compose/docker_compose_iceberg_lakekeeper_catalog.yml +++ b/tests/integration/compose/docker_compose_iceberg_lakekeeper_catalog.yml @@ -70,3 +70,27 @@ services: retries: 5 start_period: 10s cpus: 3 + + mock-oauth: + image: python:3.12-alpine + command: + - python + - -c + - | + from http.server import HTTPServer, BaseHTTPRequestHandler + import json + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_token() + def do_POST(self): + self.send_token() + def send_token(self): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + payload = {"access_token": "test-token", "expires_in": 3600, "token_type": "Bearer"} + self.wfile.write(json.dumps(payload).encode()) + def log_message(self, format, *args): + pass + HTTPServer(("0.0.0.0", 9999), Handler).serve_forever() + cpus: 1 diff --git a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py index c11966d21895..e53da5599c66 100644 --- a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py +++ b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py @@ -19,6 +19,7 @@ from helpers.test_tools import TSV, csv_compare BASE_URL = "http://lakekeeper:8181/catalog" +MOCK_OAUTH_URL = "http://mock-oauth:9999/token" CATALOG_NAME = "demo" WAREHOUSE_NAME = "demo" @@ -474,3 +475,250 @@ def test_invalid_auth_header_format(started_cluster): ) assert "Invalid auth header format" in str(err.value) +<<<<<<< HEAD +======= + +def get_credentials_profile_events(node, query_id): + node.query("SYSTEM FLUSH LOGS") + vended = int(node.query( + f"SELECT ProfileEvents['DataLakeRestCatalogCredentialsVended'] " + f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + )) + hits = int(node.query( + f"SELECT ProfileEvents['DataLakeRestCatalogCredentialsCacheHits'] " + f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + )) + return vended, hits + + +def get_auth_token_profile_events(node, query_id): + node.query("SYSTEM FLUSH LOGS") + refreshed = int(node.query( + f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenRefreshed'] " + f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + )) + cache_hits = int(node.query( + f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenCacheHits'] " + f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + )) + return refreshed, cache_hits + + +def test_auth_token_profile_events(started_cluster): + node = started_cluster.instances["node1"] + + test_ref = f"test_auth_token_profile_events_{uuid.uuid4().hex[:8]}" + db_name = f"{test_ref}_database" + namespace = (f"{test_ref}_namespace",) + table_name = f"{test_ref}_table" + + catalog = load_catalog_impl(started_cluster) + if namespace not in catalog.list_namespaces(): + catalog.create_namespace(namespace) + + schema = Schema( + NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), + NestedField(field_id=2, name="data", field_type=StringType(), required=False), + ) + catalog.create_table( + namespace + (table_name,), + schema=schema, + properties={"write.metadata.compression-codec": "none"}, + ) + + # The catalog client is initialized lazily on the first database access, + # not during CREATE DATABASE. OAuth credentials must use client_id:client_secret + # format; oauth_server_uri points to a mock token endpoint in docker compose. + create_clickhouse_iceberg_database( + started_cluster, + node, + db_name, + additional_settings={ + "catalog_credential": "test:secret", + "oauth_server_uri": MOCK_OAUTH_URL, + }, + ) + + qid1 = f"{test_ref}-show-1-{uuid.uuid4()}" + node.query(f"SHOW TABLES FROM {db_name}", query_id=qid1) + assert table_name in node.query(f"SHOW TABLES FROM {db_name}") + refreshed, cache_hits = get_auth_token_profile_events(node, qid1) + assert refreshed >= 1 + + qid2 = f"{test_ref}-show-2-{uuid.uuid4()}" + node.query(f"SHOW TABLES FROM {db_name}", query_id=qid2) + refreshed, cache_hits = get_auth_token_profile_events(node, qid2) + assert refreshed == 0 and cache_hits >= 1 + + +def test_vended_credentials_cache(started_cluster): + node = started_cluster.instances["node1"] + catalog = load_catalog_impl(started_cluster) + + test_ref = f"test_vended_credentials_cache_{uuid.uuid4().hex[:8]}" + namespace = (f"{test_ref}_namespace",) + table_name = f"{test_ref}_table" + db_name = f"{test_ref}_database" + + +def create_int_table(catalog, namespace, table_name, rows=1): + if namespace not in catalog.list_namespaces(): + catalog.create_namespace(namespace) + schema = Schema( + NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), + NestedField(field_id=2, name="data", field_type=StringType(), required=False), + ) + table = catalog.create_table( + namespace + (table_name,), + schema=schema, + properties={"write.metadata.compression-codec": "none"}, + ) + table.append( + pa.Table.from_pandas( + pd.DataFrame({"id": list(range(rows)), "data": ["x"] * rows}).astype( + {"id": "int32"} + ) + ) + ) + + +def test_vended_credentials_cache(started_cluster): + node = started_cluster.instances["node1"] + catalog = load_catalog_impl(started_cluster) + + test_ref = f"test_vended_credentials_cache_{uuid.uuid4().hex[:8]}" + namespace = (f"{test_ref}_namespace",) + table_name = f"{test_ref}_table" + db_name = f"{test_ref}_database" + + create_int_table(catalog, namespace, table_name) + + query = f"SELECT count() FROM {db_name}.`{namespace[0]}.{table_name}`" + + # Caching enabled (default TTL): the second query reuses cached credentials + # and does not ask the catalog to vend them again. + create_clickhouse_iceberg_database(started_cluster, node, db_name) + + qid = f"{test_ref}-cache-1-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, _ = get_credentials_profile_events(node, qid) + assert vended >= 1 + + qid = f"{test_ref}-cache-2-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, hits = get_credentials_profile_events(node, qid) + assert vended == 0 and hits >= 1 + + # Caching disabled (TTL = 0): every query asks the catalog to vend credentials. + create_clickhouse_iceberg_database( + started_cluster, node, db_name, + additional_settings={"vended_credentials_cache_ttl": 0}, + ) + + qid = f"{test_ref}-nocache-1-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, hits = get_credentials_profile_events(node, qid) + assert vended >= 1 and hits == 0 + + qid = f"{test_ref}-nocache-2-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, hits = get_credentials_profile_events(node, qid) + assert vended >= 1 and hits == 0 + + +def test_vended_credentials_cache_invalidated_on_table_replace(started_cluster): + node = started_cluster.instances["node1"] + catalog = load_catalog_impl(started_cluster) + + test_ref = f"test_vended_credentials_cache_replace_{uuid.uuid4().hex[:8]}" + namespace = (f"{test_ref}_namespace",) + table_name = f"{test_ref}_table" + db_name = f"{test_ref}_database" + + create_int_table(catalog, namespace, table_name) + create_clickhouse_iceberg_database(started_cluster, node, db_name) + query = f"SELECT count() FROM {db_name}.`{namespace[0]}.{table_name}`" + + # Populate the cache, then confirm the next query reuses it. + node.query(query, query_id=f"{test_ref}-1-{uuid.uuid4()}") + qid = f"{test_ref}-2-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, hits = get_credentials_profile_events(node, qid) + assert vended == 0 and hits >= 1 + + # Replace the table (new UUID and location) while the cache entry is still valid. + catalog.drop_table(namespace + (table_name,)) + create_int_table(catalog, namespace, table_name, rows=2) + + # The stale entry must be detected, so credentials are re-vended and the new table is read. + qid = f"{test_ref}-3-{uuid.uuid4()}" + assert node.query(query, query_id=qid).strip() == "2" + vended, _ = get_credentials_profile_events(node, qid) + assert vended >= 1 + + # The re-vended credentials are cached under the new identity, so the next query reuses them. + qid = f"{test_ref}-4-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, hits = get_credentials_profile_events(node, qid) + assert vended == 0 and hits >= 1 + + +@pytest.mark.skip( + reason="ALTER DATABASE ... MODIFY SETTING is not supported for the DataLakeCatalog " + "engine on this branch: DatabaseDataLake does not override " + "IDatabase::applySettingsChanges, and the RestCatalog CatalogState / " + "prepareSettingsChanges / commitSettingsChanges machinery this test relies on is " + "upstream code that is not part of this PR and has not been backported here, so " + "the ALTER fails with NOT_IMPLEMENTED before the cache behaviour is ever exercised. " + "The credentials cache itself is covered by test_vended_credentials_cache and " + "test_vended_credentials_cache_invalidated_on_table_replace. Re-enable this test " + "together with the ALTER DATABASE ... MODIFY SETTING backport." +) +def test_vended_credentials_cache_cleared_on_auth_change(started_cluster): + node = started_cluster.instances["node1"] + catalog = load_catalog_impl(started_cluster) + + test_ref = f"test_vended_credentials_cache_auth_{uuid.uuid4().hex[:8]}" + namespace = (f"{test_ref}_namespace",) + table_name = f"{test_ref}_table" + db_name = f"{test_ref}_database" + + create_int_table(catalog, namespace, table_name) + + # Header-mode database: `auth_header` is the only auth setting that can be altered. + node.query(f"DROP DATABASE IF EXISTS {db_name}") + node.query( + f""" + CREATE DATABASE {db_name} + ENGINE = DataLakeCatalog('{BASE_URL}') + SETTINGS + catalog_type = 'rest', + warehouse = 'demo', + storage_endpoint = 'http://minio1:9001/warehouse-rest', + auth_header = 'Authorization: Bearer initial_dummy' + """, + settings={"allow_experimental_database_iceberg": 1}, + ) + + query = f"SELECT count() FROM {db_name}.`{namespace[0]}.{table_name}`" + + # Populate the cache and confirm the next query reuses it. + node.query(query, query_id=f"{test_ref}-1-{uuid.uuid4()}") + qid = f"{test_ref}-2-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, hits = get_credentials_profile_events(node, qid) + assert vended == 0 and hits >= 1 + + # Committing new auth settings must drop the cached credentials. + node.query( + f"ALTER DATABASE {db_name} MODIFY SETTING auth_header = 'Authorization: Bearer altered_dummy'" + ) + + qid = f"{test_ref}-3-{uuid.uuid4()}" + node.query(query, query_id=qid) + vended, _ = get_credentials_profile_events(node, qid) + assert vended >= 1 + + node.query(f"DROP DATABASE IF EXISTS {db_name}") + +>>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) From 18782d3ab013662db787cf03b0b78badad2ba600 Mon Sep 17 00:00:00 2001 From: Andrey Zvonov <32552679+zvonand@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:58:27 +0200 Subject: [PATCH 4/4] Resolve conflicts in cherry-pick of #2222 Threaded the new `used_cached_oauth_token` out-parameter through antalya-26.8's `getAuthHeaders(const CatalogState &, bool)` shape instead of the source PR's `(bool, method, url, extra_headers, body)` shape, and kept antalya-26.8's S3TablesCatalog network-primitive overrides (it has no `getAuthHeaders` override to re-sign). Adapted: getAuthHeaders signature - antalya-26.8 re-signatured the virtual to take a CatalogState snapshot and dropped the method/url/extra_headers/body parameters Adapted: OneLakeCatalog::getAuthHeaders override - antalya-26.8-only override had to grow the new parameter to keep overriding Adapted: S3TablesCatalog - antalya-26.8 overrides createReadBuffer/sendRequest instead of getAuthHeaders, so the source PR's signature change there is a no-op Adapted: RestCatalog::sendRequest - kept antalya-26.8's X-Iceberg-Access-Delegation header and withOutCallbackFixedContentLength while adopting the PR's create_buffer/retry restructuring Adapted: added the IntegerType pyiceberg import the source PR's new test needs (present on the source branch via tests not on antalya-26.8) Source-PR: #2222 (https://github.com/Altinity/ClickHouse/pull/2222) Adapted: test_auth_token_profile_events - the source PR's helper read ProfileEvents names ('...AuthTokenRefreshed', '...AuthTokenCacheHits') that the PR's own ProfileEvents.cpp does not define; switched to the events the ported code emits ('...AuthTokenRetrieve', '...AuthTokenCachedValid') Adapted: test_auth_token_profile_events - DatabaseDataLake on antalya-26.8 builds the catalog eagerly on CREATE DATABASE (only ATTACH is lazy), so the first token fetch is attributed to the CREATE query; the assertions now check the retrieve on the CREATE query and cache hits on the following SHOW TABLES queries Adapted: dropped the body-less test_vended_credentials_cache stub - upstream it is shadowed by the full test of the vended-credentials cache, which is not part of this PR and is not on antalya-26.8, so here it would have been a no-op test Adapted: gtest_s3_uri.cpp IOTestS3URI.ResolveS3Endpoint - antalya-26.8's expandRegionToAmazonPath() resolves the endpoint through the AWS SDK endpoint provider, which aborts (aws-c-common: "allocator && aws_byte_cursor_is_valid") unless the SDK has been initialized; the test now initializes it via S3::ClientFactory::instance() (as gtest_aws_s3_client.cpp does) so it passes when run on its own instead of only after some other test happened to init the SDK. --- src/Databases/DataLake/RestCatalog.cpp | 82 ++----- src/Databases/DataLake/RestCatalog.h | 28 +-- src/Databases/DataLake/S3TablesCatalog.cpp | 10 - src/Databases/DataLake/S3TablesCatalog.h | 10 - src/IO/S3/tests/gtest_s3_uri.cpp | 8 + .../test.py | 223 ++---------------- 6 files changed, 55 insertions(+), 306 deletions(-) diff --git a/src/Databases/DataLake/RestCatalog.cpp b/src/Databases/DataLake/RestCatalog.cpp index d85a983a7d95..9c15ff5774b6 100644 --- a/src/Databases/DataLake/RestCatalog.cpp +++ b/src/Databases/DataLake/RestCatalog.cpp @@ -323,17 +323,10 @@ void RestCatalog::validateAuthHeaders(const DB::HTTPHeaderEntry & header) const getContext()->getGlobalContext()->getHTTPHeaderFilter().checkAndNormalizeHeaders(header_to_check); } -<<<<<<< HEAD -DB::HTTPHeaderEntries RestCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const -======= DB::HTTPHeaderEntries RestCatalog::getAuthHeaders( + const CatalogState & catalog_state, bool update_token, - const String & /*method*/, - const Poco::URI & /*url*/, - const DB::HTTPHeaderEntries & /*extra_headers*/, - const String & /*body*/, bool * used_cached_oauth_token) const ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { fiu_do_on(DB::FailPoints::check_database_datalake_negative, { @@ -546,7 +539,10 @@ void RestCatalog::applySettingsChangesToState( } } -DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const +DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders( + const CatalogState & catalog_state, + bool update_token, + bool * used_cached_oauth_token) const { DB::HTTPHeaderEntries headers; if (!catalog_state.refresh_token.empty()) @@ -556,7 +552,7 @@ DB::HTTPHeaderEntries OneLakeCatalog::getAuthHeaders(const CatalogState & catalo } else { - headers = RestCatalog::getAuthHeaders(catalog_state, update_token); + headers = RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); } headers.emplace_back("User-Agent", fmt::format("ClickHouse/{}{} OneLake-Catalog", VERSION_STRING, VERSION_OFFICIAL)); return headers; @@ -1045,17 +1041,10 @@ BigLakeCatalog::BigLakeCatalog( state.set(std::make_unique(std::move(initial_state))); } -<<<<<<< HEAD -DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders(const CatalogState & catalog_state, bool update_token) const -======= DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( + const CatalogState & catalog_state, bool update_token, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, bool * used_cached_oauth_token) const ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { /// Google Cloud OAuth2 for BigLake. /// Uses GCP metadata service or Application Default Credentials to get access token. @@ -1094,11 +1083,7 @@ DB::HTTPHeaderEntries BigLakeCatalog::getAuthHeaders( return headers; } -<<<<<<< HEAD - return RestCatalog::getAuthHeaders(catalog_state, update_token); -======= - return RestCatalog::getAuthHeaders(update_token, method, url, extra_headers, body, used_cached_oauth_token); ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) + return RestCatalog::getAuthHeaders(catalog_state, update_token, used_cached_oauth_token); } AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() const @@ -1120,24 +1105,10 @@ AccessToken BigLakeCatalog::retrieveGoogleCloudAccessTokenFromRefreshToken() con AccessToken BigLakeCatalog::retrieveGoogleCloudAccessToken() const { -<<<<<<< HEAD - const auto & context = getContext(); -======= ProfileEvents::increment(ProfileEvents::DataLakeRestCatalogAuthTokenRetrieve); auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::DataLakeRestCatalogAuthTokenRefreshedMicroseconds); - if (!google_adc_client_id.empty() && !google_adc_client_secret.empty() && !google_adc_refresh_token.empty()) - { - try - { - return retrieveGoogleCloudAccessTokenFromRefreshToken(); - } - catch (const DB::Exception & e) - { - LOG_DEBUG(log, "Failed to use ADC credentials, falling back to metadata service: {}", e.what()); - } - } ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) + const auto & context = getContext(); /// An explicit Application Default Credentials triple is a user-supplied credential, so it is honored. /// Fail closed if it does not work: do not fall back to the server's GCP metadata service, which would @@ -1270,11 +1241,7 @@ DB::ReadWriteBufferFromHTTPPtr RestCatalog::createReadBuffer( auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { -<<<<<<< HEAD - auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token); -======= - auto result_headers = getAuthHeaders(update_token, Poco::Net::HTTPRequest::HTTP_GET, url, headers, {}, &used_cached_oauth_token); ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) + auto result_headers = auth_headers ? *auth_headers : getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); std::move(headers.begin(), headers.end(), std::back_inserter(result_headers)); return DB::BuilderRWBufferFromHTTP(url) @@ -1869,10 +1836,6 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & LOG_TEST(log, "REST catalog {} {} body ({} bytes): {}", method, endpoint, body_str.size(), body_str); - DB::HTTPHeaderEntries headers = getAuthHeaders(catalog_state, /* update_token = */ true); - headers.emplace_back("Content-Type", "application/json"); - headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); - const auto & context = getContext(); DB::ReadWriteBufferFromHTTP::OutStreamCallback out_stream_callback; @@ -1887,28 +1850,11 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & /// enable_url_encoding=false to allow using tables with encoded sequences in names like 'foo%2Fbar' Poco::URI url(endpoint, /* enable_url_encoding */ false); -<<<<<<< HEAD - auto wb = DB::BuilderRWBufferFromHTTP(url) - .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) - .withMethod(method) - .withSettings(context->getReadSettings()) - .withTimeouts(DB::ConnectionTimeouts::getHTTPTimeouts(context->getSettingsRef(), context->getServerSettings())) - .withHostFilter(&context->getRemoteHostFilter()) - .withHeaders(headers) - .withOutCallback(out_stream_callback) - /// Send the JSON body with an explicit Content-Length: Snowflake Horizon rejects - /// chunked transfer encoding on catalog commits with HTTP 500 and an empty body. - .withOutCallbackFixedContentLength(body_str.size()) - .withSkipNotFound(false) - .create(credentials); -======= - DB::HTTPHeaderEntries extra_headers; - extra_headers.emplace_back("Content-Type", "application/json"); - auto create_buffer = [&](bool update_token, bool & used_cached_oauth_token) { - DB::HTTPHeaderEntries headers = getAuthHeaders(update_token, method, url, extra_headers, body_str, &used_cached_oauth_token); + DB::HTTPHeaderEntries headers = getAuthHeaders(catalog_state, update_token, &used_cached_oauth_token); headers.emplace_back("Content-Type", "application/json"); + headers.emplace_back("X-Iceberg-Access-Delegation", "vended-credentials"); return DB::BuilderRWBufferFromHTTP(url) .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP) .withMethod(method) @@ -1917,10 +1863,12 @@ void RestCatalog::sendRequest(const CatalogState & catalog_state, const String & .withHostFilter(&context->getRemoteHostFilter()) .withHeaders(headers) .withOutCallback(out_stream_callback) + /// Send the JSON body with an explicit Content-Length: Snowflake Horizon rejects + /// chunked transfer encoding on catalog commits with HTTP 500 and an empty body. + .withOutCallbackFixedContentLength(body_str.size()) .withSkipNotFound(false) .create(credentials); }; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) try { diff --git a/src/Databases/DataLake/RestCatalog.h b/src/Databases/DataLake/RestCatalog.h index c33c3feb9da3..4444c3eed5f6 100644 --- a/src/Databases/DataLake/RestCatalog.h +++ b/src/Databases/DataLake/RestCatalog.h @@ -205,20 +205,12 @@ class RestCatalog : public ICatalog, public DB::WithContext const std::string & table_name, TableMetadata & result) const; -<<<<<<< HEAD /// Load catalog config (special http handler) utilizing information from catalog_state and auth_headers. Config loadConfig(const CatalogState & catalog_state, const std::optional & auth_headers = std::nullopt); - virtual DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const; -======= - Config loadConfig(); virtual DB::HTTPHeaderEntries getAuthHeaders( + const CatalogState & catalog_state, bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) + bool * used_cached_oauth_token) const; void validateAuthHeaders(const DB::HTTPHeaderEntry & header) const; @@ -290,7 +282,10 @@ class OneLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_ONELAKE; } - DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const override; + DB::HTTPHeaderEntries getAuthHeaders( + const CatalogState & catalog_state, + bool update_token, + bool * used_cached_oauth_token) const override; static void validateSettingsChanges(const DB::SettingsChanges & changes, AuthMode auth_mode); @@ -337,17 +332,10 @@ class BigLakeCatalog : public RestCatalog return DB::DatabaseDataLakeCatalogType::ICEBERG_BIGLAKE; } -<<<<<<< HEAD - DB::HTTPHeaderEntries getAuthHeaders(const CatalogState & catalog_state, bool update_token) const override; -======= DB::HTTPHeaderEntries getAuthHeaders( + const CatalogState & catalog_state, bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) + bool * used_cached_oauth_token) const override; const std::string & getGoogleADCClientId() const { return google_adc_client_id; } const std::string & getGoogleADCClientSecret() const { return google_adc_client_secret; } diff --git a/src/Databases/DataLake/S3TablesCatalog.cpp b/src/Databases/DataLake/S3TablesCatalog.cpp index 1b3d9152d2de..bbf1365250b5 100644 --- a/src/Databases/DataLake/S3TablesCatalog.cpp +++ b/src/Databases/DataLake/S3TablesCatalog.cpp @@ -261,17 +261,7 @@ void S3TablesCatalog::dropTable(const String & namespace_name, const String & ta } } -<<<<<<< HEAD namespace -======= -DB::HTTPHeaderEntries S3TablesCatalog::getAuthHeaders( - bool /*update_token*/, - const String & method, - const Poco::URI & url, - const DB::HTTPHeaderEntries & extra_headers, - const String & body, - bool * /*used_cached_oauth_token*/) const ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) { /// `signRequestWithAWSV4` returns the full set of headers that the AWS SDK kept on diff --git a/src/Databases/DataLake/S3TablesCatalog.h b/src/Databases/DataLake/S3TablesCatalog.h index d5d93f0f35dc..65ca66500ed1 100644 --- a/src/Databases/DataLake/S3TablesCatalog.h +++ b/src/Databases/DataLake/S3TablesCatalog.h @@ -48,7 +48,6 @@ class S3TablesCatalog final : public RestCatalog void dropTable(const String & namespace_name, const String & table_name, bool delete_data) const override; protected: -<<<<<<< HEAD /// Override the network primitives instead of `getAuthHeaders` so the SigV4 signer has /// access to the final URL, method, and request body for canonicalisation. /// `catalog_state` and `auth_headers` are unused here: authentication is derived from the @@ -66,15 +65,6 @@ class S3TablesCatalog final : public RestCatalog Poco::JSON::Object::Ptr request_body, const String & method, bool ignore_result) const override; -======= - DB::HTTPHeaderEntries getAuthHeaders( - bool update_token, - const String & method = {}, - const Poco::URI & url = {}, - const DB::HTTPHeaderEntries & extra_headers = {}, - const String & body = {}, - bool * used_cached_oauth_token = nullptr) const override; ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events) private: const String region; diff --git a/src/IO/S3/tests/gtest_s3_uri.cpp b/src/IO/S3/tests/gtest_s3_uri.cpp index 5fce6f693d25..92556b442884 100644 --- a/src/IO/S3/tests/gtest_s3_uri.cpp +++ b/src/IO/S3/tests/gtest_s3_uri.cpp @@ -6,6 +6,8 @@ #if USE_AWS_S3 +#include + TEST(IOTestS3URI, PathStyleNoKey) { using namespace DB; @@ -42,6 +44,12 @@ TEST(IOTestS3URI, ResolveS3Endpoint) { using namespace DB; + /// expandRegionToAmazonPath() goes through the SDK's endpoint provider, which requires + /// the AWS SDK (and its CRT allocator) to be initialized. In the server this happens + /// implicitly because every S3 client is created through ClientFactory; here the test may + /// be the only thing running, so initialize it explicitly. + S3::ClientFactory::instance(); + ASSERT_EQ(S3::expandRegionToAmazonPath("us-east-1"), "https://s3.us-east-1.amazonaws.com"); ASSERT_EQ(S3::expandRegionToAmazonPath("eu-west-1"), diff --git a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py index e53da5599c66..d8ac831a78d8 100644 --- a/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py +++ b/tests/integration/test_database_iceberg_lakekeeper_catalog/test.py @@ -9,6 +9,7 @@ from pyiceberg.schema import Schema from pyiceberg.types import ( DoubleType, + IntegerType, NestedField, StringType, ) @@ -475,30 +476,15 @@ def test_invalid_auth_header_format(started_cluster): ) assert "Invalid auth header format" in str(err.value) -<<<<<<< HEAD -======= - -def get_credentials_profile_events(node, query_id): - node.query("SYSTEM FLUSH LOGS") - vended = int(node.query( - f"SELECT ProfileEvents['DataLakeRestCatalogCredentialsVended'] " - f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" - )) - hits = int(node.query( - f"SELECT ProfileEvents['DataLakeRestCatalogCredentialsCacheHits'] " - f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" - )) - return vended, hits - def get_auth_token_profile_events(node, query_id): node.query("SYSTEM FLUSH LOGS") refreshed = int(node.query( - f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenRefreshed'] " + f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenRetrieve'] " f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" )) cache_hits = int(node.query( - f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenCacheHits'] " + f"SELECT ProfileEvents['DataLakeRestCatalogAuthTokenCachedValid'] " f"FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" )) return refreshed, cache_hits @@ -526,199 +512,38 @@ def test_auth_token_profile_events(started_cluster): properties={"write.metadata.compression-codec": "none"}, ) - # The catalog client is initialized lazily on the first database access, - # not during CREATE DATABASE. OAuth credentials must use client_id:client_secret - # format; oauth_server_uri points to a mock token endpoint in docker compose. - create_clickhouse_iceberg_database( - started_cluster, - node, - db_name, - additional_settings={ - "catalog_credential": "test:secret", - "oauth_server_uri": MOCK_OAUTH_URL, - }, - ) - - qid1 = f"{test_ref}-show-1-{uuid.uuid4()}" - node.query(f"SHOW TABLES FROM {db_name}", query_id=qid1) - assert table_name in node.query(f"SHOW TABLES FROM {db_name}") - refreshed, cache_hits = get_auth_token_profile_events(node, qid1) - assert refreshed >= 1 - - qid2 = f"{test_ref}-show-2-{uuid.uuid4()}" - node.query(f"SHOW TABLES FROM {db_name}", query_id=qid2) - refreshed, cache_hits = get_auth_token_profile_events(node, qid2) - assert refreshed == 0 and cache_hits >= 1 - - -def test_vended_credentials_cache(started_cluster): - node = started_cluster.instances["node1"] - catalog = load_catalog_impl(started_cluster) - - test_ref = f"test_vended_credentials_cache_{uuid.uuid4().hex[:8]}" - namespace = (f"{test_ref}_namespace",) - table_name = f"{test_ref}_table" - db_name = f"{test_ref}_database" - - -def create_int_table(catalog, namespace, table_name, rows=1): - if namespace not in catalog.list_namespaces(): - catalog.create_namespace(namespace) - schema = Schema( - NestedField(field_id=1, name="id", field_type=IntegerType(), required=False), - NestedField(field_id=2, name="data", field_type=StringType(), required=False), - ) - table = catalog.create_table( - namespace + (table_name,), - schema=schema, - properties={"write.metadata.compression-codec": "none"}, - ) - table.append( - pa.Table.from_pandas( - pd.DataFrame({"id": list(range(rows)), "data": ["x"] * rows}).astype( - {"id": "int32"} - ) - ) - ) - - -def test_vended_credentials_cache(started_cluster): - node = started_cluster.instances["node1"] - catalog = load_catalog_impl(started_cluster) - - test_ref = f"test_vended_credentials_cache_{uuid.uuid4().hex[:8]}" - namespace = (f"{test_ref}_namespace",) - table_name = f"{test_ref}_table" - db_name = f"{test_ref}_database" - - create_int_table(catalog, namespace, table_name) - - query = f"SELECT count() FROM {db_name}.`{namespace[0]}.{table_name}`" - - # Caching enabled (default TTL): the second query reuses cached credentials - # and does not ask the catalog to vend them again. - create_clickhouse_iceberg_database(started_cluster, node, db_name) - - qid = f"{test_ref}-cache-1-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, _ = get_credentials_profile_events(node, qid) - assert vended >= 1 - - qid = f"{test_ref}-cache-2-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, hits = get_credentials_profile_events(node, qid) - assert vended == 0 and hits >= 1 - - # Caching disabled (TTL = 0): every query asks the catalog to vend credentials. - create_clickhouse_iceberg_database( - started_cluster, node, db_name, - additional_settings={"vended_credentials_cache_ttl": 0}, - ) - - qid = f"{test_ref}-nocache-1-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, hits = get_credentials_profile_events(node, qid) - assert vended >= 1 and hits == 0 - - qid = f"{test_ref}-nocache-2-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, hits = get_credentials_profile_events(node, qid) - assert vended >= 1 and hits == 0 - - -def test_vended_credentials_cache_invalidated_on_table_replace(started_cluster): - node = started_cluster.instances["node1"] - catalog = load_catalog_impl(started_cluster) - - test_ref = f"test_vended_credentials_cache_replace_{uuid.uuid4().hex[:8]}" - namespace = (f"{test_ref}_namespace",) - table_name = f"{test_ref}_table" - db_name = f"{test_ref}_database" - - create_int_table(catalog, namespace, table_name) - create_clickhouse_iceberg_database(started_cluster, node, db_name) - query = f"SELECT count() FROM {db_name}.`{namespace[0]}.{table_name}`" - - # Populate the cache, then confirm the next query reuses it. - node.query(query, query_id=f"{test_ref}-1-{uuid.uuid4()}") - qid = f"{test_ref}-2-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, hits = get_credentials_profile_events(node, qid) - assert vended == 0 and hits >= 1 - - # Replace the table (new UUID and location) while the cache entry is still valid. - catalog.drop_table(namespace + (table_name,)) - create_int_table(catalog, namespace, table_name, rows=2) - - # The stale entry must be detected, so credentials are re-vended and the new table is read. - qid = f"{test_ref}-3-{uuid.uuid4()}" - assert node.query(query, query_id=qid).strip() == "2" - vended, _ = get_credentials_profile_events(node, qid) - assert vended >= 1 - - # The re-vended credentials are cached under the new identity, so the next query reuses them. - qid = f"{test_ref}-4-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, hits = get_credentials_profile_events(node, qid) - assert vended == 0 and hits >= 1 - - -@pytest.mark.skip( - reason="ALTER DATABASE ... MODIFY SETTING is not supported for the DataLakeCatalog " - "engine on this branch: DatabaseDataLake does not override " - "IDatabase::applySettingsChanges, and the RestCatalog CatalogState / " - "prepareSettingsChanges / commitSettingsChanges machinery this test relies on is " - "upstream code that is not part of this PR and has not been backported here, so " - "the ALTER fails with NOT_IMPLEMENTED before the cache behaviour is ever exercised. " - "The credentials cache itself is covered by test_vended_credentials_cache and " - "test_vended_credentials_cache_invalidated_on_table_replace. Re-enable this test " - "together with the ALTER DATABASE ... MODIFY SETTING backport." -) -def test_vended_credentials_cache_cleared_on_auth_change(started_cluster): - node = started_cluster.instances["node1"] - catalog = load_catalog_impl(started_cluster) - - test_ref = f"test_vended_credentials_cache_auth_{uuid.uuid4().hex[:8]}" - namespace = (f"{test_ref}_namespace",) - table_name = f"{test_ref}_table" - db_name = f"{test_ref}_database" - - create_int_table(catalog, namespace, table_name) - - # Header-mode database: `auth_header` is the only auth setting that can be altered. + # The catalog client is built eagerly by CREATE DATABASE (only ATTACH defers it), + # so the first access token is fetched by that query. OAuth credentials must use + # client_id:client_secret format; oauth_server_uri points to a mock token endpoint + # in docker compose. node.query(f"DROP DATABASE IF EXISTS {db_name}") + qid_create = f"{test_ref}-create-{uuid.uuid4()}" node.query( f""" CREATE DATABASE {db_name} - ENGINE = DataLakeCatalog('{BASE_URL}') + ENGINE = DataLakeCatalog('{BASE_URL}', 'minio', '{minio_secret_key}') SETTINGS catalog_type = 'rest', warehouse = 'demo', storage_endpoint = 'http://minio1:9001/warehouse-rest', - auth_header = 'Authorization: Bearer initial_dummy' + catalog_credential = 'test:secret', + oauth_server_uri = '{MOCK_OAUTH_URL}' """, + query_id=qid_create, settings={"allow_experimental_database_iceberg": 1}, ) + refreshed, _ = get_auth_token_profile_events(node, qid_create) + assert refreshed >= 1 - query = f"SELECT count() FROM {db_name}.`{namespace[0]}.{table_name}`" - - # Populate the cache and confirm the next query reuses it. - node.query(query, query_id=f"{test_ref}-1-{uuid.uuid4()}") - qid = f"{test_ref}-2-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, hits = get_credentials_profile_events(node, qid) - assert vended == 0 and hits >= 1 - - # Committing new auth settings must drop the cached credentials. - node.query( - f"ALTER DATABASE {db_name} MODIFY SETTING auth_header = 'Authorization: Bearer altered_dummy'" - ) - - qid = f"{test_ref}-3-{uuid.uuid4()}" - node.query(query, query_id=qid) - vended, _ = get_credentials_profile_events(node, qid) - assert vended >= 1 + # Every later catalog request reuses the cached token instead of fetching a new one. + qid1 = f"{test_ref}-show-1-{uuid.uuid4()}" + node.query(f"SHOW TABLES FROM {db_name}", query_id=qid1) + assert table_name in node.query(f"SHOW TABLES FROM {db_name}") + refreshed, cache_hits = get_auth_token_profile_events(node, qid1) + assert refreshed == 0 and cache_hits >= 1 - node.query(f"DROP DATABASE IF EXISTS {db_name}") + qid2 = f"{test_ref}-show-2-{uuid.uuid4()}" + node.query(f"SHOW TABLES FROM {db_name}", query_id=qid2) + refreshed, cache_hits = get_auth_token_profile_events(node, qid2) + assert refreshed == 0 and cache_hits >= 1 ->>>>>>> a88ca756219 (Merge pull request #2222 from Altinity/feature/antalya-26.6/datalake-catalog-auth-token-profile-events)