From 6c8bac7e8de4accc1ed2999629f4fe029c6416ea Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Tue, 8 Sep 2026 11:42:20 +0100 Subject: [PATCH 1/8] feat(encryption/kms): add AWS KMS client --- Cargo.lock | 255 +++++++++++++++-- Cargo.toml | 6 + README.md | 7 + crates/kms/aws/Cargo.toml | 46 ++++ crates/kms/aws/DEPENDENCIES.rust.tsv | 393 +++++++++++++++++++++++++++ crates/kms/aws/LICENSE | 1 + crates/kms/aws/NOTICE | 1 + crates/kms/aws/README.md | 26 ++ crates/kms/aws/public-api.txt | 23 ++ crates/kms/aws/src/client.rs | 390 ++++++++++++++++++++++++++ crates/kms/aws/src/config.rs | 147 ++++++++++ crates/kms/aws/src/factory.rs | 215 +++++++++++++++ crates/kms/aws/src/lib.rs | 85 ++++++ 13 files changed, 1576 insertions(+), 19 deletions(-) create mode 100644 crates/kms/aws/Cargo.toml create mode 100644 crates/kms/aws/DEPENDENCIES.rust.tsv create mode 120000 crates/kms/aws/LICENSE create mode 120000 crates/kms/aws/NOTICE create mode 100644 crates/kms/aws/README.md create mode 100644 crates/kms/aws/public-api.txt create mode 100644 crates/kms/aws/src/client.rs create mode 100644 crates/kms/aws/src/config.rs create mode 100644 crates/kms/aws/src/factory.rs create mode 100644 crates/kms/aws/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 010f8985eb..df631a3718 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -698,6 +698,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-kms" +version = "1.111.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62d6d9b9c60e08be622a6bfc98e483d75683795caaf9704886970aa07ed8529" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json", + "aws-smithy-observability 0.2.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.5.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-s3tables" version = "1.59.0" @@ -876,22 +901,32 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" +checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" dependencies = [ "aws-smithy-async", + "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", - "h2", + "bytes", + "h2 0.3.27", + "h2 0.4.16", + "http 0.2.12", "http 1.5.0", - "hyper", + "http-body 0.4.6", + "http-body 1.0.1", + "hyper 0.14.32", + "hyper 1.10.1", "hyper-rustls", "hyper-util", + "indexmap 2.14.0", "pin-project-lite", "rustls", "rustls-native-certs", "rustls-pki-types", + "serde", + "serde_json", "tokio", "tokio-rustls", "tower", @@ -909,6 +944,18 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-mocks" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0922f8ee1ce1f90f0baf9c9036f566161547a7ab9f09571d61d56455ba35598" +dependencies = [ + "aws-smithy-http-client", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + [[package]] name = "aws-smithy-observability" version = "0.2.6" @@ -927,6 +974,25 @@ dependencies = [ "aws-smithy-runtime-api", ] +[[package]] +name = "aws-smithy-protocol-test" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a" +dependencies = [ + "assert-json-diff", + "aws-smithy-runtime-api", + "base64-simd", + "cbor-diag", + "ciborium", + "http 0.2.12", + "pretty_assertions", + "regex-lite", + "roxmltree", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "aws-smithy-query" version = "0.60.15" @@ -961,13 +1027,14 @@ dependencies = [ "pin-utils", "tokio", "tracing", + "tracing-subscriber", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.13.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" +checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -1016,9 +1083,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.1" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", @@ -1333,6 +1400,25 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbor-diag" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" +dependencies = [ + "bs58", + "chrono", + "data-encoding", + "half", + "nom", + "num-bigint 0.4.8", + "num-rational", + "num-traits", + "separator", + "url", + "uuid", +] + [[package]] name = "cc" version = "1.2.66" @@ -1398,6 +1484,33 @@ dependencies = [ "phf", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -1484,7 +1597,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1896,6 +2009,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "datafusion" version = "55.0.0" @@ -3361,6 +3480,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.16" @@ -3606,6 +3744,30 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.10.1" @@ -3616,7 +3778,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", + "h2 0.4.16", "http 1.5.0", "http-body 1.0.1", "httparse", @@ -3635,7 +3797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.5.0", - "hyper", + "hyper 1.10.1", "hyper-util", "rustls", "rustls-native-certs", @@ -3656,7 +3818,7 @@ dependencies = [ "futures-util", "http 1.5.0", "http-body 1.0.1", - "hyper", + "hyper 1.10.1", "ipnet", "libc", "percent-encoding", @@ -3927,6 +4089,19 @@ dependencies = [ "uuid", ] +[[package]] +name = "iceberg-kms-aws" +version = "0.10.1" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-kms", + "aws-smithy-mocks", + "iceberg", + "iceberg-catalog-rest", + "tokio", +] + [[package]] name = "iceberg-playground" version = "0.10.1" @@ -4698,6 +4873,12 @@ dependencies = [ "serde", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -4758,7 +4939,7 @@ dependencies = [ "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-util", "log", "pin-project-lite", @@ -4885,6 +5066,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" version = "0.4.3" @@ -4974,6 +5165,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -5060,7 +5262,7 @@ dependencies = [ "http 1.5.0", "http-body-util", "humantime", - "hyper", + "hyper 1.10.1", "itertools 0.14.0", "md-5 0.10.6", "parking_lot", @@ -6315,11 +6517,11 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2", + "h2 0.4.16", "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-rustls", "hyper-util", "js-sys", @@ -6360,7 +6562,7 @@ dependencies = [ "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-rustls", "hyper-util", "js-sys", @@ -6454,6 +6656,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + [[package]] name = "rsa" version = "0.9.10" @@ -6757,6 +6968,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "separator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" + [[package]] name = "seq-macro" version = "0.3.6" @@ -8541,7 +8758,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -8873,7 +9090,7 @@ dependencies = [ "crc32fast", "futures", "http 1.5.0", - "hyper", + "hyper 1.10.1", "lazy_static", "more-asserts", "rand 0.10.2", diff --git a/Cargo.toml b/Cargo.toml index d1083a3dc7..76766ec506 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/iceberg", "crates/integration_tests", "crates/integrations/*", + "crates/kms/*", "crates/property-macro", "crates/sqllogictest", "crates/storage/*", @@ -65,6 +66,10 @@ aws-sdk-glue = { version = "1.85", default-features = false, features = [ "default-https-client", "rt-tokio", ] } +aws-sdk-kms = { version = "1.111", default-features = false, features = [ + "default-https-client", + "rt-tokio", +] } aws-sdk-s3tables = { version = "1.28", default-features = false, features = [ "default-https-client", "rt-tokio", @@ -106,6 +111,7 @@ iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } iceberg-datafusion = { version = "0.10.0", path = "./crates/integrations/datafusion" } +iceberg-kms-aws = { version = "0.10.0", path = "./crates/kms/aws" } iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } indicatif = "0.18" diff --git a/README.md b/README.md index 536562b7b0..ad85690c05 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ The Apache Iceberg Rust project is composed of the following components: | [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | | [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | | [iceberg-datafusion] | [![iceberg-datafusion image]][iceberg-datafusion link] | [![docs release]][iceberg-datafusion release docs] [![docs dev]][iceberg-datafusion dev docs] | +| [iceberg-kms-aws] | [![iceberg-kms-aws image]][iceberg-kms-aws link] | [![docs release]][iceberg-kms-aws release docs] [![docs dev]][iceberg-kms-aws dev docs] | | [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue @@ -54,6 +55,12 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg-datafusion dev docs]: https://rust.iceberg.apache.org/api/iceberg_datafusion/ [iceberg-datafusion release docs]: https://docs.rs/iceberg-datafusion +[iceberg-kms-aws]: crates/kms/aws/README.md +[iceberg-kms-aws image]: https://img.shields.io/crates/v/iceberg-kms-aws.svg +[iceberg-kms-aws link]: https://crates.io/crates/iceberg-kms-aws +[iceberg-kms-aws release docs]: https://docs.rs/iceberg-kms-aws +[iceberg-kms-aws dev docs]: https://rust.iceberg.apache.org/api/iceberg_kms_aws/ + [iceberg-catalog-glue]: crates/catalog/glue/README.md [iceberg-catalog-glue image]: https://img.shields.io/crates/v/iceberg-catalog-glue.svg [iceberg-catalog-glue link]: https://crates.io/crates/iceberg-catalog-glue diff --git a/crates/kms/aws/Cargo.toml b/crates/kms/aws/Cargo.toml new file mode 100644 index 0000000000..7b43ad0bf6 --- /dev/null +++ b/crates/kms/aws/Cargo.toml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-kms-aws" +publish = true +readme = "README.md" +rust-version = { workspace = true } +version = { workspace = true } + +categories = ["database"] +description = "AWS KMS support for Apache Iceberg client-side encryption" +keywords = ["iceberg", "aws", "kms", "encryption"] +license = { workspace = true } +repository = { workspace = true } + +[dependencies] +async-trait = { workspace = true } +aws-config = { workspace = true } +aws-sdk-kms = { workspace = true } +iceberg = { workspace = true } + +[dev-dependencies] +aws-sdk-kms = { workspace = true, features = ["test-util"] } +aws-smithy-mocks = "0.3" +iceberg-catalog-rest = { workspace = true } +tokio = { workspace = true, features = ["macros"] } + +[lints] +workspace = true diff --git a/crates/kms/aws/DEPENDENCIES.rust.tsv b/crates/kms/aws/DEPENDENCIES.rust.tsv new file mode 100644 index 0000000000..ac08de2646 --- /dev/null +++ b/crates/kms/aws/DEPENDENCIES.rust.tsv @@ -0,0 +1,393 @@ +crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib +adler2@2.0.1 X X X +aead@0.5.2 X X +aes@0.8.4 X X +aes-gcm@0.10.3 X X +ahash@0.8.12 X X +aho-corasick@1.1.4 X X +alloc-no-stdlib@2.0.4 X +alloc-stdlib@0.2.4 X +android_system_properties@0.1.5 X X +anyhow@1.0.103 X X +apache-avro@0.21.0 X +arc-swap@1.9.2 X X +array-init@2.1.0 X X +arrow-arith@59.2.0 X +arrow-array@59.2.0 X X +arrow-buffer@59.2.0 X +arrow-cast@59.2.0 X +arrow-data@59.2.0 X +arrow-ipc@59.2.0 X +arrow-ord@59.2.0 X +arrow-schema@59.2.0 X +arrow-select@59.2.0 X +arrow-string@59.2.0 X +as-any@0.3.2 X X +assert-json-diff@2.0.2 X +async-lock@3.4.2 X X +async-trait@0.1.89 X X +atoi@2.0.0 X +atomic-waker@1.1.2 X X +autocfg@1.5.1 X X +aws-config@1.8.18 X +aws-credential-types@1.2.14 X +aws-lc-rs@1.17.1 X X +aws-lc-sys@0.42.0 X X X X X +aws-runtime@1.7.5 X +aws-sdk-kms@1.111.0 X +aws-sdk-sso@1.102.0 X +aws-sdk-ssooidc@1.104.0 X +aws-sdk-sts@1.107.0 X +aws-sigv4@1.4.5 X +aws-smithy-async@1.3.0 X +aws-smithy-http@0.63.6 X +aws-smithy-http@0.64.0 X +aws-smithy-http-client@1.4.0 X +aws-smithy-json@0.62.7 X +aws-smithy-observability@0.2.6 X +aws-smithy-observability@0.3.0 X +aws-smithy-protocol-test@0.64.0 X +aws-smithy-query@0.60.15 X +aws-smithy-runtime@1.12.0 X +aws-smithy-runtime-api@1.16.0 X +aws-smithy-runtime-api-macros@1.1.0 X +aws-smithy-schema@0.1.0 X +aws-smithy-schema@0.2.0 X +aws-smithy-types@1.6.3 X +aws-smithy-xml@0.60.15 X +aws-types@1.3.16 X +backon@1.6.0 X +base64@0.22.1 X X +base64@0.23.0 X X +base64-simd@0.8.0 X +bigdecimal@0.4.10 X X +bimap@0.6.3 X X +bitflags@2.13.0 X X +block-buffer@0.10.4 X X +block-buffer@0.12.1 X X +bnum@0.12.1 X X +bon@3.9.3 X X +bon-macros@3.9.3 X X +brotli@8.0.4 X X +brotli-decompressor@5.0.3 X X +bs58@0.5.1 X X +bumpalo@3.20.3 X X +bytemuck@1.25.0 X X X +bytemuck_derive@1.10.2 X X X +byteorder@1.5.0 X X +bytes@1.12.0 X +bytes-utils@0.1.4 X X +cbor-diag@0.1.12 X X +cc@1.2.66 X X +cfg-if@1.0.4 X X +chrono@0.4.45 X X +ciborium@0.2.2 X +ciborium-io@0.2.2 X +ciborium-ll@0.2.2 X +cipher@0.4.4 X X +cmake@0.1.58 X X +cmov@0.5.4 X X +concurrent-queue@2.5.0 X X +const-oid@0.10.2 X X +const-random@0.1.18 X X +const-random-macro@0.1.16 X X +core-foundation@0.10.1 X X +core-foundation-sys@0.8.7 X X +cpufeatures@0.2.17 X X +cpufeatures@0.3.0 X X +crc32fast@1.5.0 X X +crossbeam-channel@0.5.16 X X +crossbeam-epoch@0.9.20 X X +crossbeam-utils@0.8.22 X X +crunchy@0.2.4 X +crypto-common@0.1.7 X X +crypto-common@0.2.2 X X +ctr@0.9.2 X X +ctutils@0.4.2 X X +darling@0.20.11 X +darling@0.23.0 X +darling_core@0.20.11 X +darling_core@0.23.0 X +darling_macro@0.20.11 X +darling_macro@0.23.0 X +data-encoding@2.11.1 X +deranged@0.5.8 X X +derive_builder@0.20.2 X X +derive_builder_core@0.20.2 X X +derive_builder_macro@0.20.2 X X +diff@0.1.13 X X +digest@0.10.7 X X +digest@0.11.3 X X +displaydoc@0.2.6 X X +dissimilar@1.0.11 X +dunce@1.0.5 X X X +either@1.16.0 X X +equivalent@1.0.2 X X +erased-serde@0.4.10 X X +errno@0.3.14 X X +event-listener@5.4.1 X X +event-listener-strategy@0.5.4 X X +expect-test@1.5.1 X X +fastnum@0.7.5 X X +fastrand@2.4.1 X X +find-msvc-tools@0.1.9 X X +flatbuffers@25.12.19 X +flate2@1.1.9 X X +fnv@1.0.7 X X +form_urlencoded@1.2.2 X X +fs_extra@1.3.0 X +futures@0.3.32 X X +futures-channel@0.3.32 X X +futures-core@0.3.32 X X +futures-executor@0.3.32 X X +futures-io@0.3.32 X X +futures-macro@0.3.32 X X +futures-sink@0.3.32 X X +futures-task@0.3.32 X X +futures-util@0.3.32 X X +generic-array@0.14.7 X +getrandom@0.2.17 X X +getrandom@0.3.4 X X +getrandom@0.4.3 X X +ghash@0.5.1 X X +gloo-timers@0.3.0 X X +h2@0.3.27 X +h2@0.4.16 X +half@2.7.1 X X +hashbrown@0.17.1 X X +heck@0.5.0 X X +hex@0.4.3 X X +hmac@0.13.0 X X +http@0.2.12 X X +http@1.5.0 X X +http-body@0.4.6 X +http-body@1.0.1 X +http-body-util@0.1.3 X +httparse@1.10.1 X X +httpdate@1.0.3 X X +hybrid-array@0.4.13 X X +hyper@0.14.32 X +hyper@1.10.1 X +hyper-rustls@0.27.9 X X X +hyper-util@0.1.20 X +iana-time-zone@0.1.65 X X +iana-time-zone-haiku@0.1.2 X X +iceberg@0.10.1 X +iceberg-catalog-rest@0.10.1 X +iceberg-kms-aws@0.10.1 X +iceberg-property-macro@0.10.1 X +iceberg_test_utils@0.10.1 X +icu_collections@2.2.0 X +icu_locale_core@2.2.0 X +icu_normalizer@2.2.0 X +icu_normalizer_data@2.2.0 X +icu_properties@2.2.0 X +icu_properties_data@2.2.0 X +icu_provider@2.2.0 X +ident_case@1.0.1 X X +idna@1.1.0 X X +idna_adapter@1.2.2 X X +indexmap@2.14.0 X X +inout@0.1.4 X X +inventory@0.3.24 X X +ipnet@2.12.0 X X +itertools@0.13.0 X X +itoa@1.0.18 X X +jobserver@0.1.35 X X +js-sys@0.3.103 X X +lazy_static@1.5.0 X X +lexical-core@1.0.6 X X +lexical-parse-float@1.0.6 X X +lexical-parse-integer@1.0.6 X X +lexical-util@1.0.7 X X +lexical-write-float@1.0.6 X X +lexical-write-integer@1.0.6 X X +libc@0.2.186 X X +libm@0.2.16 X +litemap@0.8.2 X +lock_api@0.4.14 X X +log@0.4.33 X X +lz4_flex@0.14.0 X +matchers@0.2.0 X +memchr@2.8.2 X X +minimal-lexical@0.2.1 X X +miniz_oxide@0.8.9 X X X +mio@1.2.1 X +moka@0.12.15 X X +murmur3@0.5.2 X X +nom@7.1.3 X +nu-ansi-term@0.50.3 X +num-bigint@0.4.8 X X +num-bigint@0.5.1 X X +num-complex@0.4.6 X X +num-conv@0.2.2 X X +num-integer@0.1.46 X X +num-rational@0.4.2 X X +num-traits@0.2.19 X X +once_cell@1.21.4 X X +opaque-debug@0.3.1 X X +openssl-probe@0.2.1 X X +ordered-float@4.6.0 X +outref@0.5.2 X +parking@2.2.1 X X +parking_lot@0.12.5 X X +parking_lot_core@0.9.12 X X +parquet@59.2.0 X +percent-encoding@2.3.2 X X +pin-project-lite@0.2.17 X X +pin-utils@0.1.0 X X +pkg-config@0.3.33 X X +polyval@0.6.2 X X +portable-atomic@1.13.1 X X +potential_utf@0.1.5 X +powerfmt@0.2.0 X X +ppv-lite86@0.2.21 X X +pretty_assertions@1.4.1 X X +prettyplease@0.2.37 X X +proc-macro2@1.0.106 X X +quad-rand@0.2.3 X +quote@1.0.46 X X +r-efi@5.3.0 X X X +r-efi@6.0.0 X X X +rand@0.9.5 X X +rand_chacha@0.9.0 X X +rand_core@0.6.4 X X +rand_core@0.9.5 X X +redox_syscall@0.5.18 X +regex@1.13.1 X X +regex-automata@0.4.16 X X +regex-lite@0.1.9 X X +regex-syntax@0.8.11 X X +reqwest@0.12.28 X X +ring@0.17.14 X X +roaring@0.11.4 X X +roxmltree@0.14.1 X X +rustc_version@0.4.1 X X +rustls@0.23.41 X X X +rustls-native-certs@0.8.4 X X X +rustls-pki-types@1.15.0 X X +rustls-webpki@0.103.13 X +rustversion@1.0.23 X X +ryu@1.0.23 X X +schannel@0.1.29 X +scopeguard@1.2.0 X X +security-framework@3.7.0 X X +security-framework-sys@2.17.0 X X +semver@1.0.28 X X +separator@0.4.1 X +seq-macro@0.3.6 X X +serde@1.0.228 X X +serde-big-array@0.5.1 X X +serde_bytes@0.11.19 X X +serde_core@1.0.228 X X +serde_derive@1.0.228 X X +serde_json@1.0.150 X X +serde_repr@0.1.20 X X +serde_urlencoded@0.7.1 X X +serde_with@3.22.0 X X +serde_with_macros@3.22.0 X X +sha1@0.10.6 X X +sha2@0.11.0 X X +sharded-slab@0.1.7 X +shlex@2.0.1 X X +signal-hook-registry@1.4.8 X X +simd-adler32@0.3.9 X +simdutf8@0.1.5 X X +slab@0.4.12 X +smallvec@1.15.2 X X +snap@1.1.1 X +socket2@0.5.10 X X +socket2@0.6.4 X X +stable_deref_trait@1.2.1 X X +strsim@0.11.1 X +strum@0.27.2 X +strum_macros@0.27.2 X +subtle@2.6.1 X +syn@2.0.119 X X +sync_wrapper@1.0.2 X +synstructure@0.13.2 X +tagptr@0.2.0 X X +thiserror@2.0.18 X X +thiserror-impl@2.0.18 X X +thread_local@1.1.9 X X +time@0.3.53 X X +time-core@0.1.9 X X +tiny-keccak@2.0.2 X +tinystr@0.8.3 X +tokio@1.53.1 X +tokio-macros@2.7.0 X +tokio-rustls@0.26.4 X X +tokio-util@0.7.18 X +tower@0.5.3 X +tower-http@0.6.11 X +tower-layer@0.3.3 X +tower-service@0.3.3 X +tracing@0.1.44 X +tracing-attributes@0.1.31 X +tracing-core@0.1.36 X +tracing-log@0.2.0 X +tracing-serde@0.2.0 X +tracing-subscriber@0.3.23 X +try-lock@0.2.5 X +twox-hash@2.1.2 X +typed-builder@0.20.1 X X +typed-builder-macro@0.20.1 X X +typeid@1.0.3 X X +typenum@1.20.1 X X +typetag@0.2.22 X X +typetag-impl@0.2.22 X X +unicode-ident@1.0.24 X X X +universal-hash@0.5.1 X X +untrusted@0.9.0 X +url@2.5.8 X X +urlencoding@2.1.3 X +utf8_iter@1.0.4 X X +uuid@1.26.0 X X +version_check@0.9.5 X X +vsimd@0.8.0 X +want@0.3.1 X +wasi@0.11.1+wasi-snapshot-preview1 X X X +wasip2@1.0.4+wasi-0.2.12 X X X +wasm-bindgen@0.2.126 X X +wasm-bindgen-futures@0.4.76 X X +wasm-bindgen-macro@0.2.126 X X +wasm-bindgen-macro-support@0.2.126 X X +wasm-bindgen-shared@0.2.126 X X +wasm-streams@0.4.2 X X +web-sys@0.3.103 X X +windows-core@0.62.2 X X +windows-implement@0.60.2 X X +windows-interface@0.59.3 X X +windows-link@0.2.1 X X +windows-result@0.4.1 X X +windows-strings@0.5.1 X X +windows-sys@0.52.0 X X +windows-sys@0.61.2 X X +windows-targets@0.52.6 X X +windows_aarch64_gnullvm@0.52.6 X X +windows_aarch64_msvc@0.52.6 X X +windows_i686_gnu@0.52.6 X X +windows_i686_gnullvm@0.52.6 X X +windows_i686_msvc@0.52.6 X X +windows_x86_64_gnu@0.52.6 X X +windows_x86_64_gnullvm@0.52.6 X X +windows_x86_64_msvc@0.52.6 X X +wit-bindgen@0.57.1 X X X +writeable@0.6.3 X +xmlparser@0.13.6 X X +yansi@1.0.1 X X +yoke@0.8.3 X +yoke-derive@0.8.2 X +zerocopy@0.8.53 X X X +zerocopy-derive@0.8.53 X X X +zerofrom@0.1.8 X +zerofrom-derive@0.1.7 X +zeroize@1.9.0 X X +zerotrie@0.2.4 X +zerovec@0.11.6 X +zerovec-derive@0.11.3 X +zlib-rs@0.6.5 X +zmij@1.0.21 X +zstd@0.13.3 X +zstd-safe@7.2.4 X X +zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/kms/aws/LICENSE b/crates/kms/aws/LICENSE new file mode 120000 index 0000000000..5853aaea53 --- /dev/null +++ b/crates/kms/aws/LICENSE @@ -0,0 +1 @@ +../../../LICENSE \ No newline at end of file diff --git a/crates/kms/aws/NOTICE b/crates/kms/aws/NOTICE new file mode 120000 index 0000000000..295f6bdb3a --- /dev/null +++ b/crates/kms/aws/NOTICE @@ -0,0 +1 @@ +../../../NOTICE \ No newline at end of file diff --git a/crates/kms/aws/README.md b/crates/kms/aws/README.md new file mode 100644 index 0000000000..d72643ec07 --- /dev/null +++ b/crates/kms/aws/README.md @@ -0,0 +1,26 @@ + + +# Apache Iceberg AWS KMS + +This crate provides AWS Key Management Service support for Apache Iceberg +client-side encryption. + +See the [API documentation](https://docs.rs/iceberg-kms-aws/latest) for +configuration and usage. diff --git a/crates/kms/aws/public-api.txt b/crates/kms/aws/public-api.txt new file mode 100644 index 0000000000..7025896db2 --- /dev/null +++ b/crates/kms/aws/public-api.txt @@ -0,0 +1,23 @@ +pub mod iceberg_kms_aws +pub struct iceberg_kms_aws::AwsKmsClientFactory +impl iceberg_kms_aws::AwsKmsClientFactory +pub fn iceberg_kms_aws::AwsKmsClientFactory::new() -> Self +pub fn iceberg_kms_aws::AwsKmsClientFactory::with_sdk_config(self, sdk_config: aws_types::sdk_config::SdkConfig) -> Self +impl core::clone::Clone for iceberg_kms_aws::AwsKmsClientFactory +pub fn iceberg_kms_aws::AwsKmsClientFactory::clone(&self) -> iceberg_kms_aws::AwsKmsClientFactory +impl core::default::Default for iceberg_kms_aws::AwsKmsClientFactory +pub fn iceberg_kms_aws::AwsKmsClientFactory::default() -> iceberg_kms_aws::AwsKmsClientFactory +impl core::fmt::Debug for iceberg_kms_aws::AwsKmsClientFactory +pub fn iceberg_kms_aws::AwsKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::kms::factory::KmsClientFactory for iceberg_kms_aws::AwsKmsClientFactory +pub fn iceberg_kms_aws::AwsKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub const iceberg_kms_aws::AWS_ACCESS_KEY_ID: &str +pub const iceberg_kms_aws::AWS_PROFILE_NAME: &str +pub const iceberg_kms_aws::AWS_REGION_NAME: &str +pub const iceberg_kms_aws::AWS_SECRET_ACCESS_KEY: &str +pub const iceberg_kms_aws::AWS_SESSION_TOKEN: &str +pub const iceberg_kms_aws::KMS_DATA_KEY_SPEC: &str +pub const iceberg_kms_aws::KMS_DATA_KEY_SPEC_DEFAULT: &str +pub const iceberg_kms_aws::KMS_ENCRYPTION_ALGORITHM_SPEC: &str +pub const iceberg_kms_aws::KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str +pub const iceberg_kms_aws::KMS_ENDPOINT: &str diff --git a/crates/kms/aws/src/client.rs b/crates/kms/aws/src/client.rs new file mode 100644 index 0000000000..3abd93cd3e --- /dev/null +++ b/crates/kms/aws/src/client.rs @@ -0,0 +1,390 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt; + +use async_trait::async_trait; +use aws_sdk_kms::Client; +use aws_sdk_kms::primitives::Blob; +use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; +use iceberg::encryption::{GeneratedKey, KeyManagementClient, SensitiveBytes}; +use iceberg::{Error, ErrorKind, Result}; + +pub(crate) struct AwsKeyManagementClient { + client: Client, + encryption_algorithm: EncryptionAlgorithmSpec, + data_key_spec: DataKeySpec, +} + +impl AwsKeyManagementClient { + pub(crate) fn new( + client: Client, + encryption_algorithm: EncryptionAlgorithmSpec, + data_key_spec: DataKeySpec, + ) -> Self { + Self { + client, + encryption_algorithm, + data_key_spec, + } + } + + fn missing_response_field(operation: &str, field: &str) -> Error { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS {operation} response did not contain {field}"), + ) + } +} + +impl fmt::Debug for AwsKeyManagementClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AwsKeyManagementClient") + .field("encryption_algorithm", &self.encryption_algorithm) + .field("data_key_spec", &self.data_key_spec) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl KeyManagementClient for AwsKeyManagementClient { + async fn wrap_key(&self, key: &[u8], wrapping_key_id: &str) -> Result> { + let response = self + .client + .encrypt() + .key_id(wrapping_key_id) + .encryption_algorithm(self.encryption_algorithm.clone()) + .plaintext(Blob::new(key)) + .send() + .await + .map_err(|source| { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS Encrypt failed for key '{wrapping_key_id}'"), + ) + .with_source(source) + })?; + + response + .ciphertext_blob + .map(Blob::into_inner) + .ok_or_else(|| Self::missing_response_field("Encrypt", "ciphertext_blob")) + } + + async fn unwrap_key( + &self, + wrapped_key: &[u8], + wrapping_key_id: &str, + ) -> Result { + let response = self + .client + .decrypt() + .key_id(wrapping_key_id) + .encryption_algorithm(self.encryption_algorithm.clone()) + .ciphertext_blob(Blob::new(wrapped_key)) + .send() + .await + .map_err(|source| { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS Decrypt failed for key '{wrapping_key_id}'"), + ) + .with_source(source) + })?; + + response + .plaintext + .map(|blob| SensitiveBytes::new(blob.into_inner())) + .ok_or_else(|| Self::missing_response_field("Decrypt", "plaintext")) + } + + fn supports_key_generation(&self) -> bool { + self.encryption_algorithm == EncryptionAlgorithmSpec::SymmetricDefault + } + + async fn generate_key(&self, wrapping_key_id: &str) -> Result { + let response = self + .client + .generate_data_key() + .key_id(wrapping_key_id) + .key_spec(self.data_key_spec.clone()) + .send() + .await + .map_err(|source| { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS GenerateDataKey failed for key '{wrapping_key_id}'"), + ) + .with_source(source) + })?; + + let plaintext = response + .plaintext + .map(|blob| SensitiveBytes::new(blob.into_inner())) + .ok_or_else(|| Self::missing_response_field("GenerateDataKey", "plaintext"))?; + let wrapped_key = response + .ciphertext_blob + .map(Blob::into_inner) + .ok_or_else(|| Self::missing_response_field("GenerateDataKey", "ciphertext_blob"))?; + + Ok(GeneratedKey::new(plaintext, wrapped_key)) + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + use std::sync::{Arc, Mutex}; + + use aws_sdk_kms::Client; + use aws_sdk_kms::operation::decrypt::DecryptOutput; + use aws_sdk_kms::operation::encrypt::{EncryptError, EncryptOutput}; + use aws_sdk_kms::operation::generate_data_key::GenerateDataKeyOutput; + use aws_sdk_kms::types::error::NotFoundException; + use aws_smithy_mocks::{RuleMode, mock, mock_client}; + use iceberg::encryption::{EncryptionManager, StandardKeyMetadata}; + + use super::*; + + const KEY_ID: &str = "arn:aws:kms:eu-west-2:123456789012:key/test-key"; + + #[tokio::test] + async fn test_wrap_key() { + let plaintext = b"0123456789abcdef"; + let ciphertext = b"wrapped-key"; + let rule = mock!(Client::encrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::SymmetricDefault) + && request.plaintext().map(AsRef::as_ref) == Some(plaintext.as_slice()) + }) + .then_output(move || { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(ciphertext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + assert_eq!(kms.wrap_key(plaintext, KEY_ID).await.unwrap(), ciphertext); + assert_eq!(rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_unwrap_key() { + let plaintext = b"0123456789abcdef"; + let ciphertext = b"wrapped-key"; + let rule = mock!(Client::decrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::SymmetricDefault) + && request.ciphertext_blob().map(AsRef::as_ref) == Some(ciphertext.as_slice()) + }) + .then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + assert_eq!( + kms.unwrap_key(ciphertext, KEY_ID).await.unwrap().as_bytes(), + plaintext + ); + assert_eq!(rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_generate_key() { + let plaintext = b"0123456789abcdef0123456789abcdef"; + let ciphertext = b"generated-wrapped-key"; + let rule = mock!(Client::generate_data_key) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) && request.key_spec() == Some(&DataKeySpec::Aes256) + }) + .then_output(move || { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new(plaintext)) + .ciphertext_blob(Blob::new(ciphertext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + let generated = kms.generate_key(KEY_ID).await.unwrap(); + assert_eq!(generated.key().as_bytes(), plaintext); + assert_eq!(generated.wrapped_key(), ciphertext); + assert_eq!(rule.num_calls(), 1); + } + + #[test] + fn test_key_generation_support_depends_on_encryption_algorithm() { + let client = mock_client!(aws_sdk_kms, &[]); + + assert!( + AwsKeyManagementClient::new( + client.clone(), + EncryptionAlgorithmSpec::SymmetricDefault, + DataKeySpec::Aes256, + ) + .supports_key_generation() + ); + assert!( + !AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::RsaesOaepSha256, + DataKeySpec::Aes256, + ) + .supports_key_generation() + ); + } + + #[tokio::test] + async fn test_asymmetric_manager_roundtrip_uses_encrypt_instead_of_generate_data_key() { + let plaintext_kek = Arc::new(Mutex::new(Vec::new())); + let plaintext_kek_from_encrypt = Arc::clone(&plaintext_kek); + let encrypt_rule = mock!(Client::encrypt) + .match_requests(move |request| { + if request.key_id() != Some(KEY_ID) + || request.encryption_algorithm() + != Some(&EncryptionAlgorithmSpec::RsaesOaepSha256) + { + return false; + } + + *plaintext_kek_from_encrypt.lock().unwrap() = request + .plaintext() + .expect("EncryptionManager must provide a plaintext KEK") + .as_ref() + .to_vec(); + true + }) + .then_output(|| { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(b"wrapped-asymmetric-kek")) + .build() + }); + let plaintext_kek_for_decrypt = Arc::clone(&plaintext_kek); + let decrypt_rule = mock!(Client::decrypt).then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext_kek_for_decrypt.lock().unwrap().clone())) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&encrypt_rule, &decrypt_rule]); + let kms: Arc = Arc::new(AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::RsaesOaepSha256, + DataKeySpec::Aes256, + )); + let manager = EncryptionManager::builder() + .kms_client(Arc::clone(&kms)) + .table_key_id(KEY_ID) + .build(); + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let encrypted_key_id = manager + .encrypt_manifest_list_key_metadata(&key_metadata) + .await + .unwrap(); + let encryption_keys = manager.with_encryption_keys(Clone::clone); + let reader = EncryptionManager::builder() + .kms_client(kms) + .table_key_id(KEY_ID) + .encryption_keys(encryption_keys) + .build(); + let decrypted = reader + .decrypt_manifest_list_key_metadata(&encrypted_key_id) + .await + .unwrap(); + + assert_eq!(decrypted, key_metadata); + assert_eq!(encrypt_rule.num_calls(), 1); + assert_eq!(decrypt_rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_aws_service_error_is_preserved_as_source() { + let rule = mock!(Client::encrypt).then_error(|| { + EncryptError::NotFoundException( + NotFoundException::builder() + .message("test key not found") + .build(), + ) + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + let error = kms.wrap_key(b"key", KEY_ID).await.unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.source().is_some()); + } + + #[tokio::test] + async fn test_missing_response_fields() { + let encrypt_rule = mock!(Client::encrypt).then_output(|| EncryptOutput::builder().build()); + let decrypt_rule = mock!(Client::decrypt).then_output(|| DecryptOutput::builder().build()); + let generate_rule = mock!(Client::generate_data_key) + .then_output(|| GenerateDataKeyOutput::builder().build()); + let client = mock_client!(aws_sdk_kms, RuleMode::Sequential, &[ + &encrypt_rule, + &decrypt_rule, + &generate_rule + ]); + let kms = test_client(client); + + let error = kms.wrap_key(b"key", KEY_ID).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("ciphertext_blob")); + + let error = kms.unwrap_key(b"wrapped", KEY_ID).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("plaintext")); + + let error = kms.generate_key(KEY_ID).await.err().unwrap(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("plaintext")); + } + + #[tokio::test] + async fn test_generate_key_requires_wrapped_key() { + let rule = mock!(Client::generate_data_key).then_output(|| { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new([0; 32])) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + let error = kms.generate_key(KEY_ID).await.err().unwrap(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("ciphertext_blob")); + } + + fn test_client(client: Client) -> AwsKeyManagementClient { + AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::SymmetricDefault, + DataKeySpec::Aes256, + ) + } +} diff --git a/crates/kms/aws/src/config.rs b/crates/kms/aws/src/config.rs new file mode 100644 index 0000000000..7377decd0f --- /dev/null +++ b/crates/kms/aws/src/config.rs @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; +use iceberg::{Error, ErrorKind, Result}; + +/// Catalog property selecting an AWS profile. +pub const AWS_PROFILE_NAME: &str = "profile_name"; +/// Catalog property selecting the AWS region. +pub const AWS_REGION_NAME: &str = "region_name"; +/// Catalog property containing a static AWS access key ID. +pub const AWS_ACCESS_KEY_ID: &str = "aws_access_key_id"; +/// Catalog property containing a static AWS secret access key. +pub const AWS_SECRET_ACCESS_KEY: &str = "aws_secret_access_key"; +/// Catalog property containing a static AWS session token. +pub const AWS_SESSION_TOKEN: &str = "aws_session_token"; + +/// Catalog property overriding the AWS KMS service endpoint. +pub const KMS_ENDPOINT: &str = "kms.endpoint"; +/// Catalog property selecting the AWS KMS encryption algorithm. +pub const KMS_ENCRYPTION_ALGORITHM_SPEC: &str = "kms.encryption-algorithm-spec"; +/// Default AWS KMS encryption algorithm, matching Iceberg Java. +pub const KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str = "SYMMETRIC_DEFAULT"; +/// Catalog property selecting the size of keys generated by AWS KMS. +pub const KMS_DATA_KEY_SPEC: &str = "kms.data-key-spec"; +/// Default AWS KMS generated key size, matching Iceberg Java. +pub const KMS_DATA_KEY_SPEC_DEFAULT: &str = "AES_256"; + +#[derive(Clone)] +pub(crate) struct AwsKmsConfig { + pub(crate) endpoint: Option, + pub(crate) encryption_algorithm: EncryptionAlgorithmSpec, + pub(crate) data_key_spec: DataKeySpec, +} + +impl AwsKmsConfig { + pub(crate) fn from_properties(properties: &HashMap) -> Result { + let encryption_algorithm = properties + .get(KMS_ENCRYPTION_ALGORITHM_SPEC) + .map(String::as_str) + .unwrap_or(KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT); + let data_key_spec = properties + .get(KMS_DATA_KEY_SPEC) + .map(String::as_str) + .unwrap_or(KMS_DATA_KEY_SPEC_DEFAULT); + + Ok(Self { + endpoint: properties.get(KMS_ENDPOINT).cloned(), + encryption_algorithm: parse_encryption_algorithm(encryption_algorithm)?, + data_key_spec: parse_data_key_spec(data_key_spec)?, + }) + } +} + +fn parse_encryption_algorithm(value: &str) -> Result { + EncryptionAlgorithmSpec::try_parse(value).map_err(|source| { + Error::new( + ErrorKind::DataInvalid, + format!("Unsupported AWS KMS encryption algorithm: {value}"), + ) + .with_source(source) + }) +} + +fn parse_data_key_spec(value: &str) -> Result { + DataKeySpec::try_parse(value).map_err(|source| { + Error::new( + ErrorKind::DataInvalid, + format!("Unsupported AWS KMS data key spec: {value}"), + ) + .with_source(source) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_defaults_match_iceberg_java() { + let config = AwsKmsConfig::from_properties(&HashMap::new()).unwrap(); + + assert_eq!( + config.encryption_algorithm, + EncryptionAlgorithmSpec::SymmetricDefault + ); + assert_eq!(config.data_key_spec, DataKeySpec::Aes256); + assert_eq!(config.endpoint, None); + } + + #[test] + fn test_parse_overrides() { + let properties = HashMap::from([ + ( + KMS_ENDPOINT.to_string(), + "http://localhost:4566".to_string(), + ), + ( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "RSAES_OAEP_SHA_256".to_string(), + ), + (KMS_DATA_KEY_SPEC.to_string(), "AES_128".to_string()), + ]); + + let config = AwsKmsConfig::from_properties(&properties).unwrap(); + assert_eq!(config.endpoint.as_deref(), Some("http://localhost:4566")); + assert_eq!( + config.encryption_algorithm, + EncryptionAlgorithmSpec::RsaesOaepSha256 + ); + assert_eq!(config.data_key_spec, DataKeySpec::Aes128); + } + + #[test] + fn test_reject_unknown_values() { + let bad_algorithm = HashMap::from([( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "unknown".to_string(), + )]); + let Err(error) = AwsKmsConfig::from_properties(&bad_algorithm) else { + panic!("unknown encryption algorithms must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + + let bad_key_spec = HashMap::from([(KMS_DATA_KEY_SPEC.to_string(), "AES_192".to_string())]); + let Err(error) = AwsKmsConfig::from_properties(&bad_key_spec) else { + panic!("unknown data-key specifications must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/kms/aws/src/factory.rs b/crates/kms/aws/src/factory.rs new file mode 100644 index 0000000000..32c5c77bdd --- /dev/null +++ b/crates/kms/aws/src/factory.rs @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use async_trait::async_trait; +use aws_config::{BehaviorVersion, SdkConfig}; +use aws_sdk_kms::Client as KmsClient; +use aws_sdk_kms::config::{Builder as KmsConfigBuilder, Credentials, Region}; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::{Error, ErrorKind, Result}; + +use crate::client::AwsKeyManagementClient; +use crate::config::{ + AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + AwsKmsConfig, +}; + +/// Factory for AWS KMS-backed Iceberg key management clients. +/// +/// By default, AWS credentials and region are resolved through the AWS SDK's +/// standard provider chains. Catalog properties can override the profile, +/// region, or static credentials. Applications with custom credential or HTTP +/// providers can supply a complete [`SdkConfig`] through +/// [`with_sdk_config`](Self::with_sdk_config). +#[derive(Clone, Default)] +pub struct AwsKmsClientFactory { + sdk_config: Option, +} + +impl AwsKmsClientFactory { + /// Create a factory using catalog properties and the AWS SDK default chains. + pub fn new() -> Self { + Self::default() + } + + /// Use an application-provided AWS SDK configuration. + /// + /// When set, SDK-wide credential, region, retry, and HTTP configuration + /// properties are not rebuilt from catalog properties. KMS-specific + /// properties such as `kms.endpoint` continue to be applied. + pub fn with_sdk_config(mut self, sdk_config: SdkConfig) -> Self { + self.sdk_config = Some(sdk_config); + self + } +} + +impl fmt::Debug for AwsKmsClientFactory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AwsKmsClientFactory") + .field("has_sdk_config", &self.sdk_config.is_some()) + .finish() + } +} + +#[async_trait] +impl KmsClientFactory for AwsKmsClientFactory { + async fn create_kms_client( + &self, + properties: &HashMap, + ) -> Result> { + let kms_config = AwsKmsConfig::from_properties(properties)?; + let sdk_config = match &self.sdk_config { + Some(config) => config.clone(), + None => create_sdk_config(properties).await?, + }; + + let mut builder = KmsConfigBuilder::from(&sdk_config); + if sdk_config.behavior_version().is_none() { + builder = builder.behavior_version(BehaviorVersion::latest()); + } + if let Some(endpoint) = &kms_config.endpoint { + builder = builder.endpoint_url(endpoint); + } + + Ok(Arc::new(AwsKeyManagementClient::new( + KmsClient::from_conf(builder.build()), + kms_config.encryption_algorithm, + kms_config.data_key_spec, + ))) + } +} + +async fn create_sdk_config(properties: &HashMap) -> Result { + let mut loader = aws_config::defaults(BehaviorVersion::latest()); + + match ( + properties.get(AWS_ACCESS_KEY_ID), + properties.get(AWS_SECRET_ACCESS_KEY), + ) { + (Some(access_key), Some(secret_key)) => { + let credentials = Credentials::new( + access_key, + secret_key, + properties.get(AWS_SESSION_TOKEN).cloned(), + None, + "catalog-properties", + ); + loader = loader.credentials_provider(credentials); + } + (None, None) => { + if properties.contains_key(AWS_SESSION_TOKEN) { + return Err(incomplete_credentials_error()); + } + } + _ => return Err(incomplete_credentials_error()), + } + + if let Some(profile) = properties.get(AWS_PROFILE_NAME) { + loader = loader.profile_name(profile); + } + if let Some(region) = properties.get(AWS_REGION_NAME) { + loader = loader.region(Region::new(region.clone())); + } + + Ok(loader.load().await) +} + +fn incomplete_credentials_error() -> Error { + Error::new( + ErrorKind::DataInvalid, + format!( + "AWS static credentials require both '{AWS_ACCESS_KEY_ID}' and '{AWS_SECRET_ACCESS_KEY}'" + ), + ) +} + +#[cfg(test)] +mod tests { + use aws_config::SdkConfig; + use aws_sdk_kms::config::ProvideCredentials; + use iceberg::encryption::kms::KmsClientFactory; + + use super::*; + use crate::KMS_ENDPOINT; + + #[tokio::test] + async fn test_create_client_with_injected_sdk_config() { + let factory = AwsKmsClientFactory::new().with_sdk_config(test_sdk_config()); + let properties = HashMap::from([( + KMS_ENDPOINT.to_string(), + "http://localhost:4566".to_string(), + )]); + + let client = factory.create_kms_client(&properties).await.unwrap(); + assert!(client.supports_key_generation()); + } + + #[tokio::test] + async fn test_create_client_adds_missing_behavior_version() { + let factory = AwsKmsClientFactory::new().with_sdk_config(SdkConfig::builder().build()); + + let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + + assert!(client.supports_key_generation()); + } + + #[tokio::test] + async fn test_reject_incomplete_static_credentials() { + let only_access_key = + HashMap::from([(AWS_ACCESS_KEY_ID.to_string(), "access".to_string())]); + let error = create_sdk_config(&only_access_key).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + + let only_session_token = + HashMap::from([(AWS_SESSION_TOKEN.to_string(), "token".to_string())]); + let error = create_sdk_config(&only_session_token).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_static_credentials_and_region_override_default_chains() { + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "access".to_string()), + (AWS_SECRET_ACCESS_KEY.to_string(), "secret".to_string()), + (AWS_SESSION_TOKEN.to_string(), "token".to_string()), + (AWS_REGION_NAME.to_string(), "eu-west-2".to_string()), + ]); + + let config = create_sdk_config(&properties).await.unwrap(); + let credentials = config + .credentials_provider() + .expect("static credentials must install a credentials provider") + .provide_credentials() + .await + .unwrap(); + + assert_eq!(config.region().map(Region::as_ref), Some("eu-west-2")); + assert_eq!(credentials.access_key_id(), "access"); + assert_eq!(credentials.secret_access_key(), "secret"); + assert_eq!(credentials.session_token(), Some("token")); + } + + fn test_sdk_config() -> SdkConfig { + SdkConfig::builder() + .behavior_version(BehaviorVersion::latest()) + .build() + } +} diff --git a/crates/kms/aws/src/lib.rs b/crates/kms/aws/src/lib.rs new file mode 100644 index 0000000000..69246bc1d2 --- /dev/null +++ b/crates/kms/aws/src/lib.rs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS Key Management Service integration for Apache Iceberg client-side encryption. +//! +//! This crate provides [`AwsKmsClientFactory`], which creates AWS-backed +//! implementations of Iceberg's +//! [`KeyManagementClient`](iceberg::encryption::KeyManagementClient). The factory can +//! be supplied to any Iceberg catalog builder through +//! [`CatalogBuilder::with_kms_client_factory`](iceberg::CatalogBuilder::with_kms_client_factory). +//! +//! The KMS key referenced by a table's `encryption.key-id` property requires +//! `kms:Decrypt` for reads. Writes with `SYMMETRIC_DEFAULT` use +//! `kms:GenerateDataKey`. Writes with an asymmetric encryption algorithm +//! generate the key-encryption key locally and require `kms:Encrypt` to wrap it. +//! +//! # Configuration +//! +//! The factory uses the AWS SDK's default credential and region provider chains. +//! The `profile_name`, `region_name`, `aws_access_key_id`, +//! `aws_secret_access_key`, and `aws_session_token` catalog properties can +//! override those defaults, following the conventions used by Iceberg Rust's +//! other AWS integrations. +//! +//! Catalog properties are the properties supplied by the application when the +//! catalog is constructed. Configuration returned later by a REST catalog +//! server is not currently available when the KMS client is created. +//! +//! The following KMS properties match Iceberg Java: +//! +//! - `kms.endpoint` overrides the KMS service endpoint. +//! - `kms.encryption-algorithm-spec` selects the KMS encryption algorithm and +//! defaults to `SYMMETRIC_DEFAULT`. +//! - `kms.data-key-spec` selects the size of KMS-generated key-encryption keys +//! and defaults to `AES_256`. +//! +//! `kms.data-key-spec` is independent from the Iceberg table property +//! `encryption.data-key-length`, which controls locally generated file data keys. +//! +//! # Example +//! +//! ```rust,no_run +//! use std::collections::HashMap; +//! use std::sync::Arc; +//! +//! use iceberg::CatalogBuilder; +//! use iceberg_catalog_rest::RestCatalogBuilder; +//! use iceberg_kms_aws::AwsKmsClientFactory; +//! +//! # async fn example() -> iceberg::Result<()> { +//! let catalog = RestCatalogBuilder::default() +//! .with_kms_client_factory(Arc::new(AwsKmsClientFactory::new())) +//! .load("rest", HashMap::new()) +//! .await?; +//! # let _ = catalog; +//! # Ok(()) +//! # } +//! ``` + +#![deny(missing_docs)] + +mod client; +mod config; +mod factory; + +pub use config::{ + AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + KMS_DATA_KEY_SPEC, KMS_DATA_KEY_SPEC_DEFAULT, KMS_ENCRYPTION_ALGORITHM_SPEC, + KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT, KMS_ENDPOINT, +}; +pub use factory::AwsKmsClientFactory; From 7596b6fdee6e41beba423a89b5db6c41c086b3ef Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Tue, 8 Sep 2026 13:02:41 +0100 Subject: [PATCH 2/8] refactor(encryption/kms): use Properties derive for AWS KMS config --- Cargo.lock | 1 + crates/kms/aws/Cargo.toml | 1 + crates/kms/aws/src/config.rs | 37 +++++++++++++++--------------------- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df631a3718..9431de3950 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4099,6 +4099,7 @@ dependencies = [ "aws-smithy-mocks", "iceberg", "iceberg-catalog-rest", + "iceberg-property-macro", "tokio", ] diff --git a/crates/kms/aws/Cargo.toml b/crates/kms/aws/Cargo.toml index 7b43ad0bf6..0126147a33 100644 --- a/crates/kms/aws/Cargo.toml +++ b/crates/kms/aws/Cargo.toml @@ -35,6 +35,7 @@ async-trait = { workspace = true } aws-config = { workspace = true } aws-sdk-kms = { workspace = true } iceberg = { workspace = true } +iceberg-property-macro = { workspace = true } [dev-dependencies] aws-sdk-kms = { workspace = true, features = ["test-util"] } diff --git a/crates/kms/aws/src/config.rs b/crates/kms/aws/src/config.rs index 7377decd0f..925c1aa4f5 100644 --- a/crates/kms/aws/src/config.rs +++ b/crates/kms/aws/src/config.rs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; - use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; /// Catalog property selecting an AWS profile. pub const AWS_PROFILE_NAME: &str = "profile_name"; @@ -42,32 +41,24 @@ pub const KMS_DATA_KEY_SPEC: &str = "kms.data-key-spec"; /// Default AWS KMS generated key size, matching Iceberg Java. pub const KMS_DATA_KEY_SPEC_DEFAULT: &str = "AES_256"; -#[derive(Clone)] +#[derive(Clone, Properties)] pub(crate) struct AwsKmsConfig { + #[property(key = KMS_ENDPOINT, default = None)] pub(crate) endpoint: Option, + #[property( + key = KMS_ENCRYPTION_ALGORITHM_SPEC, + default = KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT, + parse_with = parse_encryption_algorithm + )] pub(crate) encryption_algorithm: EncryptionAlgorithmSpec, + #[property( + key = KMS_DATA_KEY_SPEC, + default = KMS_DATA_KEY_SPEC_DEFAULT, + parse_with = parse_data_key_spec + )] pub(crate) data_key_spec: DataKeySpec, } -impl AwsKmsConfig { - pub(crate) fn from_properties(properties: &HashMap) -> Result { - let encryption_algorithm = properties - .get(KMS_ENCRYPTION_ALGORITHM_SPEC) - .map(String::as_str) - .unwrap_or(KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT); - let data_key_spec = properties - .get(KMS_DATA_KEY_SPEC) - .map(String::as_str) - .unwrap_or(KMS_DATA_KEY_SPEC_DEFAULT); - - Ok(Self { - endpoint: properties.get(KMS_ENDPOINT).cloned(), - encryption_algorithm: parse_encryption_algorithm(encryption_algorithm)?, - data_key_spec: parse_data_key_spec(data_key_spec)?, - }) - } -} - fn parse_encryption_algorithm(value: &str) -> Result { EncryptionAlgorithmSpec::try_parse(value).map_err(|source| { Error::new( @@ -90,6 +81,8 @@ fn parse_data_key_spec(value: &str) -> Result { #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; #[test] From f39cbd76b8f9838cd758bb9e50ce3e9c990f73b3 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Tue, 8 Sep 2026 14:55:18 +0100 Subject: [PATCH 3/8] chore(deps): upgrade AWS SDK dependencies --- Cargo.lock | 115 ++++++++++++++++----------- Cargo.toml | 4 +- crates/kms/aws/DEPENDENCIES.rust.tsv | 29 +++---- 3 files changed, 83 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9431de3950..a89278f450 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -584,9 +584,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.18" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -594,11 +594,11 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema 0.1.0", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -615,9 +615,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -650,14 +650,14 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.5" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" +checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", - "aws-smithy-http 0.63.6", + "aws-smithy-http 0.64.0", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -684,7 +684,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-json 0.62.7", "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -700,19 +700,20 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.111.0" +version = "1.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62d6d9b9c60e08be622a6bfc98e483d75683795caaf9704886970aa07ed8529" +checksum = "c243864bc3754be9f0001414e0162fdf1370fa62c70b0cfbe1da6a6221d553c7" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -734,7 +735,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-json 0.62.7", "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -750,19 +751,20 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.102.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" +checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -775,19 +777,20 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.104.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" +checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -800,20 +803,21 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.107.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" +checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", + "aws-smithy-observability 0.3.0", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -826,12 +830,12 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.5" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", - "aws-smithy-http 0.63.6", + "aws-smithy-http 0.64.0", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -944,6 +948,17 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.63.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", + "aws-smithy-types", +] + [[package]] name = "aws-smithy-mocks" version = "0.3.0" @@ -995,19 +1010,22 @@ dependencies = [ [[package]] name = "aws-smithy-query" -version = "0.60.15" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.12.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" +checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" dependencies = [ "aws-smithy-async", "aws-smithy-http 0.64.0", @@ -1109,23 +1127,26 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.15" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.0", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.16" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema 0.1.0", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "rustc_version", "tracing", @@ -1597,7 +1618,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -7814,7 +7835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -8759,7 +8780,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 76766ec506..dc1b112672 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,12 +61,12 @@ arrow-select = "59.2" arrow-string = "59.2" as-any = "0.3.2" async-trait = "0.1.89" -aws-config = "1.8.7" +aws-config = "1.12" aws-sdk-glue = { version = "1.85", default-features = false, features = [ "default-https-client", "rt-tokio", ] } -aws-sdk-kms = { version = "1.111", default-features = false, features = [ +aws-sdk-kms = { version = "1.118", default-features = false, features = [ "default-https-client", "rt-tokio", ] } diff --git a/crates/kms/aws/DEPENDENCIES.rust.tsv b/crates/kms/aws/DEPENDENCIES.rust.tsv index ac08de2646..43756895c6 100644 --- a/crates/kms/aws/DEPENDENCIES.rust.tsv +++ b/crates/kms/aws/DEPENDENCIES.rust.tsv @@ -29,33 +29,30 @@ async-trait@0.1.89 X X atoi@2.0.0 X atomic-waker@1.1.2 X X autocfg@1.5.1 X X -aws-config@1.8.18 X -aws-credential-types@1.2.14 X +aws-config@1.12.0 X +aws-credential-types@1.3.0 X aws-lc-rs@1.17.1 X X aws-lc-sys@0.42.0 X X X X X -aws-runtime@1.7.5 X -aws-sdk-kms@1.111.0 X -aws-sdk-sso@1.102.0 X -aws-sdk-ssooidc@1.104.0 X -aws-sdk-sts@1.107.0 X -aws-sigv4@1.4.5 X +aws-runtime@1.9.2 X +aws-sdk-kms@1.118.0 X +aws-sdk-sso@1.109.0 X +aws-sdk-ssooidc@1.111.0 X +aws-sdk-sts@1.114.0 X +aws-sigv4@1.5.1 X aws-smithy-async@1.3.0 X -aws-smithy-http@0.63.6 X aws-smithy-http@0.64.0 X aws-smithy-http-client@1.4.0 X -aws-smithy-json@0.62.7 X -aws-smithy-observability@0.2.6 X +aws-smithy-json@0.63.0 X aws-smithy-observability@0.3.0 X aws-smithy-protocol-test@0.64.0 X -aws-smithy-query@0.60.15 X -aws-smithy-runtime@1.12.0 X +aws-smithy-query@0.62.0 X +aws-smithy-runtime@1.14.0 X aws-smithy-runtime-api@1.16.0 X aws-smithy-runtime-api-macros@1.1.0 X -aws-smithy-schema@0.1.0 X aws-smithy-schema@0.2.0 X aws-smithy-types@1.6.3 X -aws-smithy-xml@0.60.15 X -aws-types@1.3.16 X +aws-smithy-xml@0.62.0 X +aws-types@1.6.0 X backon@1.6.0 X base64@0.22.1 X X base64@0.23.0 X X From c4540306cb45409a969ba8a99523aa6b7f17b5cd Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Wed, 9 Sep 2026 11:45:10 +0100 Subject: [PATCH 4/8] chore(security): ignore test-only h2 advisory --- .cargo/audit.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 176d43abf6..cd60d23475 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -37,4 +37,18 @@ ignore = [ # once the transitive dependency set permits quick-xml >=0.41.0. "RUSTSEC-2026-0194", "RUSTSEC-2026-0195", + # `h2` 0.3.27 accepts an unbounded number of empty DATA frames (low-severity + # DoS). + # + # It is present only through `aws-sdk-kms/test-util` -> + # `aws-smithy-runtime/legacy-test-util`, which is required by the official + # `aws-smithy-mocks` test framework, and is absent from normal/build + # dependencies. + # + # Per SECURITY-THREAT-MODEL.md, malformed-input DoS is hardening rather than + # an Iceberg Rust trust-boundary vulnerability. + # + # Remove after the fix in + # https://github.com/smithy-lang/smithy-rs/pull/4827 reaches an AWS SDK release. + "RUSTSEC-2026-0258", ] From 25dc118f823e285ed7dcdf1b8c316b2b6e848c72 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Fri, 18 Sep 2026 10:29:16 +0100 Subject: [PATCH 5/8] refactor(kms): consolidate KMS implementations into one crate --- Cargo.lock | 2 +- Cargo.toml | 4 +- README.md | 12 +- crates/kms/{aws => }/Cargo.toml | 26 +- crates/kms/DEPENDENCIES.rust.tsv | 305 ++++++++++++++ crates/kms/LICENSE | 1 + crates/kms/NOTICE | 1 + crates/kms/{aws => }/README.md | 22 +- crates/kms/aws/DEPENDENCIES.rust.tsv | 390 ------------------ crates/kms/aws/LICENSE | 1 - crates/kms/aws/NOTICE | 1 - crates/kms/aws/public-api.txt | 23 -- crates/kms/public-api.txt | 24 ++ crates/kms/{aws/src => src/aws}/client.rs | 0 crates/kms/{aws/src => src/aws}/config.rs | 0 crates/kms/{aws/src => src/aws}/factory.rs | 6 +- crates/kms/{aws/src/lib.rs => src/aws/mod.rs} | 10 +- crates/kms/src/lib.rs | 27 ++ 18 files changed, 410 insertions(+), 445 deletions(-) rename crates/kms/{aws => }/Cargo.toml (70%) create mode 100644 crates/kms/DEPENDENCIES.rust.tsv create mode 120000 crates/kms/LICENSE create mode 120000 crates/kms/NOTICE rename crates/kms/{aws => }/README.md (59%) delete mode 100644 crates/kms/aws/DEPENDENCIES.rust.tsv delete mode 120000 crates/kms/aws/LICENSE delete mode 120000 crates/kms/aws/NOTICE delete mode 100644 crates/kms/aws/public-api.txt create mode 100644 crates/kms/public-api.txt rename crates/kms/{aws/src => src/aws}/client.rs (100%) rename crates/kms/{aws/src => src/aws}/config.rs (100%) rename crates/kms/{aws/src => src/aws}/factory.rs (98%) rename crates/kms/{aws/src/lib.rs => src/aws/mod.rs} (92%) create mode 100644 crates/kms/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a89278f450..5eafa93d46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4111,7 +4111,7 @@ dependencies = [ ] [[package]] -name = "iceberg-kms-aws" +name = "iceberg-kms" version = "0.10.1" dependencies = [ "async-trait", diff --git a/Cargo.toml b/Cargo.toml index dc1b112672..e63a1cfdc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ "crates/iceberg", "crates/integration_tests", "crates/integrations/*", - "crates/kms/*", + "crates/kms", "crates/property-macro", "crates/sqllogictest", "crates/storage/*", @@ -111,7 +111,7 @@ iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } iceberg-datafusion = { version = "0.10.0", path = "./crates/integrations/datafusion" } -iceberg-kms-aws = { version = "0.10.0", path = "./crates/kms/aws" } +iceberg-kms = { version = "0.10.0", path = "./crates/kms" } iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } indicatif = "0.18" diff --git a/README.md b/README.md index ad85690c05..4ceff3bde5 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ The Apache Iceberg Rust project is composed of the following components: | [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | | [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | | [iceberg-datafusion] | [![iceberg-datafusion image]][iceberg-datafusion link] | [![docs release]][iceberg-datafusion release docs] [![docs dev]][iceberg-datafusion dev docs] | -| [iceberg-kms-aws] | [![iceberg-kms-aws image]][iceberg-kms-aws link] | [![docs release]][iceberg-kms-aws release docs] [![docs dev]][iceberg-kms-aws dev docs] | +| [iceberg-kms] | [![iceberg-kms image]][iceberg-kms link] | [![docs release]][iceberg-kms release docs] [![docs dev]][iceberg-kms dev docs] | | [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue @@ -55,11 +55,11 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg-datafusion dev docs]: https://rust.iceberg.apache.org/api/iceberg_datafusion/ [iceberg-datafusion release docs]: https://docs.rs/iceberg-datafusion -[iceberg-kms-aws]: crates/kms/aws/README.md -[iceberg-kms-aws image]: https://img.shields.io/crates/v/iceberg-kms-aws.svg -[iceberg-kms-aws link]: https://crates.io/crates/iceberg-kms-aws -[iceberg-kms-aws release docs]: https://docs.rs/iceberg-kms-aws -[iceberg-kms-aws dev docs]: https://rust.iceberg.apache.org/api/iceberg_kms_aws/ +[iceberg-kms]: crates/kms/README.md +[iceberg-kms image]: https://img.shields.io/crates/v/iceberg-kms.svg +[iceberg-kms link]: https://crates.io/crates/iceberg-kms +[iceberg-kms release docs]: https://docs.rs/iceberg-kms +[iceberg-kms dev docs]: https://rust.iceberg.apache.org/api/iceberg_kms/ [iceberg-catalog-glue]: crates/catalog/glue/README.md [iceberg-catalog-glue image]: https://img.shields.io/crates/v/iceberg-catalog-glue.svg diff --git a/crates/kms/aws/Cargo.toml b/crates/kms/Cargo.toml similarity index 70% rename from crates/kms/aws/Cargo.toml rename to crates/kms/Cargo.toml index 0126147a33..38fbf56db6 100644 --- a/crates/kms/aws/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -18,24 +18,36 @@ [package] edition = { workspace = true } homepage = { workspace = true } -name = "iceberg-kms-aws" +name = "iceberg-kms" publish = true readme = "README.md" rust-version = { workspace = true } version = { workspace = true } categories = ["database"] -description = "AWS KMS support for Apache Iceberg client-side encryption" -keywords = ["iceberg", "aws", "kms", "encryption"] +description = "Key management service integrations for Apache Iceberg client-side encryption" +keywords = ["iceberg", "kms", "encryption"] license = { workspace = true } repository = { workspace = true } +[package.metadata.docs.rs] +all-features = true + +[features] +aws = [ + "dep:async-trait", + "dep:aws-config", + "dep:aws-sdk-kms", + "dep:iceberg-property-macro", +] +default = [] + [dependencies] -async-trait = { workspace = true } -aws-config = { workspace = true } -aws-sdk-kms = { workspace = true } +async-trait = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-sdk-kms = { workspace = true, optional = true } iceberg = { workspace = true } -iceberg-property-macro = { workspace = true } +iceberg-property-macro = { workspace = true, optional = true } [dev-dependencies] aws-sdk-kms = { workspace = true, features = ["test-util"] } diff --git a/crates/kms/DEPENDENCIES.rust.tsv b/crates/kms/DEPENDENCIES.rust.tsv new file mode 100644 index 0000000000..62c3a4564a --- /dev/null +++ b/crates/kms/DEPENDENCIES.rust.tsv @@ -0,0 +1,305 @@ +crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib +adler2@2.0.1 X X X +aead@0.5.2 X X +aes@0.8.4 X X +aes-gcm@0.10.3 X X +ahash@0.8.12 X X +aho-corasick@1.1.4 X X +alloc-no-stdlib@2.0.4 X +alloc-stdlib@0.2.4 X +android_system_properties@0.1.5 X X +anyhow@1.0.103 X X +apache-avro@0.21.0 X +array-init@2.1.0 X X +arrow-arith@59.2.0 X +arrow-array@59.2.0 X X +arrow-buffer@59.2.0 X +arrow-cast@59.2.0 X +arrow-data@59.2.0 X +arrow-ipc@59.2.0 X +arrow-ord@59.2.0 X +arrow-schema@59.2.0 X +arrow-select@59.2.0 X +arrow-string@59.2.0 X +as-any@0.3.2 X X +async-lock@3.4.2 X X +async-trait@0.1.89 X X +atoi@2.0.0 X +atomic-waker@1.1.2 X X +autocfg@1.5.1 X X +backon@1.6.0 X +base64@0.22.1 X X +base64@0.23.0 X X +bigdecimal@0.4.10 X X +bimap@0.6.3 X X +bitflags@2.13.0 X X +block-buffer@0.10.4 X X +bnum@0.12.1 X X +bon@3.9.3 X X +bon-macros@3.9.3 X X +brotli@8.0.4 X X +brotli-decompressor@5.0.3 X X +bumpalo@3.20.3 X X +bytemuck@1.25.0 X X X +bytemuck_derive@1.10.2 X X X +byteorder@1.5.0 X X +bytes@1.12.0 X +cc@1.2.66 X X +cfg-if@1.0.4 X X +chrono@0.4.45 X X +cipher@0.4.4 X X +concurrent-queue@2.5.0 X X +const-random@0.1.18 X X +const-random-macro@0.1.16 X X +core-foundation-sys@0.8.7 X X +cpufeatures@0.2.17 X X +crc32fast@1.5.0 X X +crossbeam-channel@0.5.16 X X +crossbeam-epoch@0.9.20 X X +crossbeam-utils@0.8.22 X X +crunchy@0.2.4 X +crypto-common@0.1.7 X X +ctr@0.9.2 X X +darling@0.20.11 X +darling@0.23.0 X +darling_core@0.20.11 X +darling_core@0.23.0 X +darling_macro@0.20.11 X +darling_macro@0.23.0 X +derive_builder@0.20.2 X X +derive_builder_core@0.20.2 X X +derive_builder_macro@0.20.2 X X +digest@0.10.7 X X +displaydoc@0.2.6 X X +dissimilar@1.0.11 X +either@1.16.0 X X +equivalent@1.0.2 X X +erased-serde@0.4.10 X X +event-listener@5.4.1 X X +event-listener-strategy@0.5.4 X X +expect-test@1.5.1 X X +fastnum@0.7.5 X X +fastrand@2.4.1 X X +find-msvc-tools@0.1.9 X X +flatbuffers@25.12.19 X +flate2@1.1.9 X X +fnv@1.0.7 X X +form_urlencoded@1.2.2 X X +futures@0.3.32 X X +futures-channel@0.3.32 X X +futures-core@0.3.32 X X +futures-executor@0.3.32 X X +futures-io@0.3.32 X X +futures-macro@0.3.32 X X +futures-sink@0.3.32 X X +futures-task@0.3.32 X X +futures-util@0.3.32 X X +generic-array@0.14.7 X +getrandom@0.2.17 X X +getrandom@0.3.4 X X +getrandom@0.4.3 X X +ghash@0.5.1 X X +gloo-timers@0.3.0 X X +h2@0.4.16 X +half@2.7.1 X X +hashbrown@0.17.1 X X +heck@0.5.0 X X +http@1.5.0 X X +http-body@1.0.1 X +http-body-util@0.1.3 X +httparse@1.10.1 X X +httpdate@1.0.3 X X +hyper@1.10.1 X +hyper-util@0.1.20 X +iana-time-zone@0.1.65 X X +iana-time-zone-haiku@0.1.2 X X +iceberg@0.10.1 X +iceberg-catalog-rest@0.10.1 X +iceberg-kms@0.10.1 X +iceberg-property-macro@0.10.1 X +iceberg_test_utils@0.10.1 X +icu_collections@2.2.0 X +icu_locale_core@2.2.0 X +icu_normalizer@2.2.0 X +icu_normalizer_data@2.2.0 X +icu_properties@2.2.0 X +icu_properties_data@2.2.0 X +icu_provider@2.2.0 X +ident_case@1.0.1 X X +idna@1.1.0 X X +idna_adapter@1.2.2 X X +indexmap@2.14.0 X X +inout@0.1.4 X X +inventory@0.3.24 X X +ipnet@2.12.0 X X +itertools@0.13.0 X X +itoa@1.0.18 X X +jobserver@0.1.35 X X +js-sys@0.3.103 X X +lazy_static@1.5.0 X X +lexical-core@1.0.6 X X +lexical-parse-float@1.0.6 X X +lexical-parse-integer@1.0.6 X X +lexical-util@1.0.7 X X +lexical-write-float@1.0.6 X X +lexical-write-integer@1.0.6 X X +libc@0.2.186 X X +libm@0.2.16 X +litemap@0.8.2 X +lock_api@0.4.14 X X +log@0.4.33 X X +lz4_flex@0.14.0 X +matchers@0.2.0 X +memchr@2.8.2 X X +miniz_oxide@0.8.9 X X X +mio@1.2.1 X +moka@0.12.15 X X +murmur3@0.5.2 X X +nu-ansi-term@0.50.3 X +num-bigint@0.4.8 X X +num-bigint@0.5.1 X X +num-complex@0.4.6 X X +num-integer@0.1.46 X X +num-traits@0.2.19 X X +once_cell@1.21.4 X X +opaque-debug@0.3.1 X X +ordered-float@4.6.0 X +parking@2.2.1 X X +parking_lot@0.12.5 X X +parking_lot_core@0.9.12 X X +parquet@59.2.0 X +percent-encoding@2.3.2 X X +pin-project-lite@0.2.17 X X +pkg-config@0.3.33 X X +polyval@0.6.2 X X +portable-atomic@1.13.1 X X +potential_utf@0.1.5 X +ppv-lite86@0.2.21 X X +prettyplease@0.2.37 X X +proc-macro2@1.0.106 X X +quad-rand@0.2.3 X +quote@1.0.46 X X +r-efi@5.3.0 X X X +r-efi@6.0.0 X X X +rand@0.9.5 X X +rand_chacha@0.9.0 X X +rand_core@0.6.4 X X +rand_core@0.9.5 X X +redox_syscall@0.5.18 X +regex@1.13.1 X X +regex-automata@0.4.16 X X +regex-lite@0.1.9 X X +regex-syntax@0.8.11 X X +reqwest@0.12.28 X X +ring@0.17.14 X X +roaring@0.11.4 X X +rustc_version@0.4.1 X X +rustversion@1.0.23 X X +ryu@1.0.23 X X +scopeguard@1.2.0 X X +semver@1.0.28 X X +seq-macro@0.3.6 X X +serde@1.0.228 X X +serde-big-array@0.5.1 X X +serde_bytes@0.11.19 X X +serde_core@1.0.228 X X +serde_derive@1.0.228 X X +serde_json@1.0.150 X X +serde_repr@0.1.20 X X +serde_urlencoded@0.7.1 X X +serde_with@3.22.0 X X +serde_with_macros@3.22.0 X X +sharded-slab@0.1.7 X +shlex@2.0.1 X X +simd-adler32@0.3.9 X +simdutf8@0.1.5 X X +slab@0.4.12 X +smallvec@1.15.2 X X +snap@1.1.1 X +socket2@0.6.4 X X +stable_deref_trait@1.2.1 X X +strsim@0.11.1 X +strum@0.27.2 X +strum_macros@0.27.2 X +subtle@2.6.1 X +syn@2.0.119 X X +sync_wrapper@1.0.2 X +synstructure@0.13.2 X +tagptr@0.2.0 X X +thiserror@2.0.18 X X +thiserror-impl@2.0.18 X X +thread_local@1.1.9 X X +tiny-keccak@2.0.2 X +tinystr@0.8.3 X +tokio@1.53.1 X +tokio-macros@2.7.0 X +tokio-util@0.7.18 X +tower@0.5.3 X +tower-http@0.6.11 X +tower-layer@0.3.3 X +tower-service@0.3.3 X +tracing@0.1.44 X +tracing-attributes@0.1.31 X +tracing-core@0.1.36 X +tracing-log@0.2.0 X +tracing-serde@0.2.0 X +tracing-subscriber@0.3.23 X +try-lock@0.2.5 X +twox-hash@2.1.2 X +typed-builder@0.20.1 X X +typed-builder-macro@0.20.1 X X +typeid@1.0.3 X X +typenum@1.20.1 X X +typetag@0.2.22 X X +typetag-impl@0.2.22 X X +unicode-ident@1.0.24 X X X +universal-hash@0.5.1 X X +untrusted@0.9.0 X +url@2.5.8 X X +utf8_iter@1.0.4 X X +uuid@1.26.0 X X +version_check@0.9.5 X X +want@0.3.1 X +wasi@0.11.1+wasi-snapshot-preview1 X X X +wasip2@1.0.4+wasi-0.2.12 X X X +wasm-bindgen@0.2.126 X X +wasm-bindgen-futures@0.4.76 X X +wasm-bindgen-macro@0.2.126 X X +wasm-bindgen-macro-support@0.2.126 X X +wasm-bindgen-shared@0.2.126 X X +wasm-streams@0.4.2 X X +web-sys@0.3.103 X X +windows-core@0.62.2 X X +windows-implement@0.60.2 X X +windows-interface@0.59.3 X X +windows-link@0.2.1 X X +windows-result@0.4.1 X X +windows-strings@0.5.1 X X +windows-sys@0.52.0 X X +windows-sys@0.61.2 X X +windows-targets@0.52.6 X X +windows_aarch64_gnullvm@0.52.6 X X +windows_aarch64_msvc@0.52.6 X X +windows_i686_gnu@0.52.6 X X +windows_i686_gnullvm@0.52.6 X X +windows_i686_msvc@0.52.6 X X +windows_x86_64_gnu@0.52.6 X X +windows_x86_64_gnullvm@0.52.6 X X +windows_x86_64_msvc@0.52.6 X X +wit-bindgen@0.57.1 X X X +writeable@0.6.3 X +yoke@0.8.3 X +yoke-derive@0.8.2 X +zerocopy@0.8.53 X X X +zerocopy-derive@0.8.53 X X X +zerofrom@0.1.8 X +zerofrom-derive@0.1.7 X +zeroize@1.9.0 X X +zerotrie@0.2.4 X +zerovec@0.11.6 X +zerovec-derive@0.11.3 X +zlib-rs@0.6.5 X +zmij@1.0.21 X +zstd@0.13.3 X +zstd-safe@7.2.4 X X +zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/kms/LICENSE b/crates/kms/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/crates/kms/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/kms/NOTICE b/crates/kms/NOTICE new file mode 120000 index 0000000000..fb376cfaa4 --- /dev/null +++ b/crates/kms/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/crates/kms/aws/README.md b/crates/kms/README.md similarity index 59% rename from crates/kms/aws/README.md rename to crates/kms/README.md index d72643ec07..44f77ddaf7 100644 --- a/crates/kms/aws/README.md +++ b/crates/kms/README.md @@ -17,10 +17,22 @@ ~ under the License. --> -# Apache Iceberg AWS KMS +# Apache Iceberg KMS -This crate provides AWS Key Management Service support for Apache Iceberg -client-side encryption. +This crate provides key management service integrations for Apache Iceberg +client-side encryption. Provider implementations are enabled through feature +flags. -See the [API documentation](https://docs.rs/iceberg-kms-aws/latest) for -configuration and usage. +| Provider | Feature flag | +| -------------------------- | ------------ | +| AWS Key Management Service | `aws` | + +Add the crate with the feature for the provider you need: + +```toml +[dependencies] +iceberg-kms = { version = "x.y.z", features = ["aws"] } +``` + +See the [API documentation](https://docs.rs/iceberg-kms/latest) for +provider-specific configuration and usage. diff --git a/crates/kms/aws/DEPENDENCIES.rust.tsv b/crates/kms/aws/DEPENDENCIES.rust.tsv deleted file mode 100644 index 43756895c6..0000000000 --- a/crates/kms/aws/DEPENDENCIES.rust.tsv +++ /dev/null @@ -1,390 +0,0 @@ -crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT MIT-0 Unicode-3.0 Unlicense Zlib -adler2@2.0.1 X X X -aead@0.5.2 X X -aes@0.8.4 X X -aes-gcm@0.10.3 X X -ahash@0.8.12 X X -aho-corasick@1.1.4 X X -alloc-no-stdlib@2.0.4 X -alloc-stdlib@0.2.4 X -android_system_properties@0.1.5 X X -anyhow@1.0.103 X X -apache-avro@0.21.0 X -arc-swap@1.9.2 X X -array-init@2.1.0 X X -arrow-arith@59.2.0 X -arrow-array@59.2.0 X X -arrow-buffer@59.2.0 X -arrow-cast@59.2.0 X -arrow-data@59.2.0 X -arrow-ipc@59.2.0 X -arrow-ord@59.2.0 X -arrow-schema@59.2.0 X -arrow-select@59.2.0 X -arrow-string@59.2.0 X -as-any@0.3.2 X X -assert-json-diff@2.0.2 X -async-lock@3.4.2 X X -async-trait@0.1.89 X X -atoi@2.0.0 X -atomic-waker@1.1.2 X X -autocfg@1.5.1 X X -aws-config@1.12.0 X -aws-credential-types@1.3.0 X -aws-lc-rs@1.17.1 X X -aws-lc-sys@0.42.0 X X X X X -aws-runtime@1.9.2 X -aws-sdk-kms@1.118.0 X -aws-sdk-sso@1.109.0 X -aws-sdk-ssooidc@1.111.0 X -aws-sdk-sts@1.114.0 X -aws-sigv4@1.5.1 X -aws-smithy-async@1.3.0 X -aws-smithy-http@0.64.0 X -aws-smithy-http-client@1.4.0 X -aws-smithy-json@0.63.0 X -aws-smithy-observability@0.3.0 X -aws-smithy-protocol-test@0.64.0 X -aws-smithy-query@0.62.0 X -aws-smithy-runtime@1.14.0 X -aws-smithy-runtime-api@1.16.0 X -aws-smithy-runtime-api-macros@1.1.0 X -aws-smithy-schema@0.2.0 X -aws-smithy-types@1.6.3 X -aws-smithy-xml@0.62.0 X -aws-types@1.6.0 X -backon@1.6.0 X -base64@0.22.1 X X -base64@0.23.0 X X -base64-simd@0.8.0 X -bigdecimal@0.4.10 X X -bimap@0.6.3 X X -bitflags@2.13.0 X X -block-buffer@0.10.4 X X -block-buffer@0.12.1 X X -bnum@0.12.1 X X -bon@3.9.3 X X -bon-macros@3.9.3 X X -brotli@8.0.4 X X -brotli-decompressor@5.0.3 X X -bs58@0.5.1 X X -bumpalo@3.20.3 X X -bytemuck@1.25.0 X X X -bytemuck_derive@1.10.2 X X X -byteorder@1.5.0 X X -bytes@1.12.0 X -bytes-utils@0.1.4 X X -cbor-diag@0.1.12 X X -cc@1.2.66 X X -cfg-if@1.0.4 X X -chrono@0.4.45 X X -ciborium@0.2.2 X -ciborium-io@0.2.2 X -ciborium-ll@0.2.2 X -cipher@0.4.4 X X -cmake@0.1.58 X X -cmov@0.5.4 X X -concurrent-queue@2.5.0 X X -const-oid@0.10.2 X X -const-random@0.1.18 X X -const-random-macro@0.1.16 X X -core-foundation@0.10.1 X X -core-foundation-sys@0.8.7 X X -cpufeatures@0.2.17 X X -cpufeatures@0.3.0 X X -crc32fast@1.5.0 X X -crossbeam-channel@0.5.16 X X -crossbeam-epoch@0.9.20 X X -crossbeam-utils@0.8.22 X X -crunchy@0.2.4 X -crypto-common@0.1.7 X X -crypto-common@0.2.2 X X -ctr@0.9.2 X X -ctutils@0.4.2 X X -darling@0.20.11 X -darling@0.23.0 X -darling_core@0.20.11 X -darling_core@0.23.0 X -darling_macro@0.20.11 X -darling_macro@0.23.0 X -data-encoding@2.11.1 X -deranged@0.5.8 X X -derive_builder@0.20.2 X X -derive_builder_core@0.20.2 X X -derive_builder_macro@0.20.2 X X -diff@0.1.13 X X -digest@0.10.7 X X -digest@0.11.3 X X -displaydoc@0.2.6 X X -dissimilar@1.0.11 X -dunce@1.0.5 X X X -either@1.16.0 X X -equivalent@1.0.2 X X -erased-serde@0.4.10 X X -errno@0.3.14 X X -event-listener@5.4.1 X X -event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X -fastnum@0.7.5 X X -fastrand@2.4.1 X X -find-msvc-tools@0.1.9 X X -flatbuffers@25.12.19 X -flate2@1.1.9 X X -fnv@1.0.7 X X -form_urlencoded@1.2.2 X X -fs_extra@1.3.0 X -futures@0.3.32 X X -futures-channel@0.3.32 X X -futures-core@0.3.32 X X -futures-executor@0.3.32 X X -futures-io@0.3.32 X X -futures-macro@0.3.32 X X -futures-sink@0.3.32 X X -futures-task@0.3.32 X X -futures-util@0.3.32 X X -generic-array@0.14.7 X -getrandom@0.2.17 X X -getrandom@0.3.4 X X -getrandom@0.4.3 X X -ghash@0.5.1 X X -gloo-timers@0.3.0 X X -h2@0.3.27 X -h2@0.4.16 X -half@2.7.1 X X -hashbrown@0.17.1 X X -heck@0.5.0 X X -hex@0.4.3 X X -hmac@0.13.0 X X -http@0.2.12 X X -http@1.5.0 X X -http-body@0.4.6 X -http-body@1.0.1 X -http-body-util@0.1.3 X -httparse@1.10.1 X X -httpdate@1.0.3 X X -hybrid-array@0.4.13 X X -hyper@0.14.32 X -hyper@1.10.1 X -hyper-rustls@0.27.9 X X X -hyper-util@0.1.20 X -iana-time-zone@0.1.65 X X -iana-time-zone-haiku@0.1.2 X X -iceberg@0.10.1 X -iceberg-catalog-rest@0.10.1 X -iceberg-kms-aws@0.10.1 X -iceberg-property-macro@0.10.1 X -iceberg_test_utils@0.10.1 X -icu_collections@2.2.0 X -icu_locale_core@2.2.0 X -icu_normalizer@2.2.0 X -icu_normalizer_data@2.2.0 X -icu_properties@2.2.0 X -icu_properties_data@2.2.0 X -icu_provider@2.2.0 X -ident_case@1.0.1 X X -idna@1.1.0 X X -idna_adapter@1.2.2 X X -indexmap@2.14.0 X X -inout@0.1.4 X X -inventory@0.3.24 X X -ipnet@2.12.0 X X -itertools@0.13.0 X X -itoa@1.0.18 X X -jobserver@0.1.35 X X -js-sys@0.3.103 X X -lazy_static@1.5.0 X X -lexical-core@1.0.6 X X -lexical-parse-float@1.0.6 X X -lexical-parse-integer@1.0.6 X X -lexical-util@1.0.7 X X -lexical-write-float@1.0.6 X X -lexical-write-integer@1.0.6 X X -libc@0.2.186 X X -libm@0.2.16 X -litemap@0.8.2 X -lock_api@0.4.14 X X -log@0.4.33 X X -lz4_flex@0.14.0 X -matchers@0.2.0 X -memchr@2.8.2 X X -minimal-lexical@0.2.1 X X -miniz_oxide@0.8.9 X X X -mio@1.2.1 X -moka@0.12.15 X X -murmur3@0.5.2 X X -nom@7.1.3 X -nu-ansi-term@0.50.3 X -num-bigint@0.4.8 X X -num-bigint@0.5.1 X X -num-complex@0.4.6 X X -num-conv@0.2.2 X X -num-integer@0.1.46 X X -num-rational@0.4.2 X X -num-traits@0.2.19 X X -once_cell@1.21.4 X X -opaque-debug@0.3.1 X X -openssl-probe@0.2.1 X X -ordered-float@4.6.0 X -outref@0.5.2 X -parking@2.2.1 X X -parking_lot@0.12.5 X X -parking_lot_core@0.9.12 X X -parquet@59.2.0 X -percent-encoding@2.3.2 X X -pin-project-lite@0.2.17 X X -pin-utils@0.1.0 X X -pkg-config@0.3.33 X X -polyval@0.6.2 X X -portable-atomic@1.13.1 X X -potential_utf@0.1.5 X -powerfmt@0.2.0 X X -ppv-lite86@0.2.21 X X -pretty_assertions@1.4.1 X X -prettyplease@0.2.37 X X -proc-macro2@1.0.106 X X -quad-rand@0.2.3 X -quote@1.0.46 X X -r-efi@5.3.0 X X X -r-efi@6.0.0 X X X -rand@0.9.5 X X -rand_chacha@0.9.0 X X -rand_core@0.6.4 X X -rand_core@0.9.5 X X -redox_syscall@0.5.18 X -regex@1.13.1 X X -regex-automata@0.4.16 X X -regex-lite@0.1.9 X X -regex-syntax@0.8.11 X X -reqwest@0.12.28 X X -ring@0.17.14 X X -roaring@0.11.4 X X -roxmltree@0.14.1 X X -rustc_version@0.4.1 X X -rustls@0.23.41 X X X -rustls-native-certs@0.8.4 X X X -rustls-pki-types@1.15.0 X X -rustls-webpki@0.103.13 X -rustversion@1.0.23 X X -ryu@1.0.23 X X -schannel@0.1.29 X -scopeguard@1.2.0 X X -security-framework@3.7.0 X X -security-framework-sys@2.17.0 X X -semver@1.0.28 X X -separator@0.4.1 X -seq-macro@0.3.6 X X -serde@1.0.228 X X -serde-big-array@0.5.1 X X -serde_bytes@0.11.19 X X -serde_core@1.0.228 X X -serde_derive@1.0.228 X X -serde_json@1.0.150 X X -serde_repr@0.1.20 X X -serde_urlencoded@0.7.1 X X -serde_with@3.22.0 X X -serde_with_macros@3.22.0 X X -sha1@0.10.6 X X -sha2@0.11.0 X X -sharded-slab@0.1.7 X -shlex@2.0.1 X X -signal-hook-registry@1.4.8 X X -simd-adler32@0.3.9 X -simdutf8@0.1.5 X X -slab@0.4.12 X -smallvec@1.15.2 X X -snap@1.1.1 X -socket2@0.5.10 X X -socket2@0.6.4 X X -stable_deref_trait@1.2.1 X X -strsim@0.11.1 X -strum@0.27.2 X -strum_macros@0.27.2 X -subtle@2.6.1 X -syn@2.0.119 X X -sync_wrapper@1.0.2 X -synstructure@0.13.2 X -tagptr@0.2.0 X X -thiserror@2.0.18 X X -thiserror-impl@2.0.18 X X -thread_local@1.1.9 X X -time@0.3.53 X X -time-core@0.1.9 X X -tiny-keccak@2.0.2 X -tinystr@0.8.3 X -tokio@1.53.1 X -tokio-macros@2.7.0 X -tokio-rustls@0.26.4 X X -tokio-util@0.7.18 X -tower@0.5.3 X -tower-http@0.6.11 X -tower-layer@0.3.3 X -tower-service@0.3.3 X -tracing@0.1.44 X -tracing-attributes@0.1.31 X -tracing-core@0.1.36 X -tracing-log@0.2.0 X -tracing-serde@0.2.0 X -tracing-subscriber@0.3.23 X -try-lock@0.2.5 X -twox-hash@2.1.2 X -typed-builder@0.20.1 X X -typed-builder-macro@0.20.1 X X -typeid@1.0.3 X X -typenum@1.20.1 X X -typetag@0.2.22 X X -typetag-impl@0.2.22 X X -unicode-ident@1.0.24 X X X -universal-hash@0.5.1 X X -untrusted@0.9.0 X -url@2.5.8 X X -urlencoding@2.1.3 X -utf8_iter@1.0.4 X X -uuid@1.26.0 X X -version_check@0.9.5 X X -vsimd@0.8.0 X -want@0.3.1 X -wasi@0.11.1+wasi-snapshot-preview1 X X X -wasip2@1.0.4+wasi-0.2.12 X X X -wasm-bindgen@0.2.126 X X -wasm-bindgen-futures@0.4.76 X X -wasm-bindgen-macro@0.2.126 X X -wasm-bindgen-macro-support@0.2.126 X X -wasm-bindgen-shared@0.2.126 X X -wasm-streams@0.4.2 X X -web-sys@0.3.103 X X -windows-core@0.62.2 X X -windows-implement@0.60.2 X X -windows-interface@0.59.3 X X -windows-link@0.2.1 X X -windows-result@0.4.1 X X -windows-strings@0.5.1 X X -windows-sys@0.52.0 X X -windows-sys@0.61.2 X X -windows-targets@0.52.6 X X -windows_aarch64_gnullvm@0.52.6 X X -windows_aarch64_msvc@0.52.6 X X -windows_i686_gnu@0.52.6 X X -windows_i686_gnullvm@0.52.6 X X -windows_i686_msvc@0.52.6 X X -windows_x86_64_gnu@0.52.6 X X -windows_x86_64_gnullvm@0.52.6 X X -windows_x86_64_msvc@0.52.6 X X -wit-bindgen@0.57.1 X X X -writeable@0.6.3 X -xmlparser@0.13.6 X X -yansi@1.0.1 X X -yoke@0.8.3 X -yoke-derive@0.8.2 X -zerocopy@0.8.53 X X X -zerocopy-derive@0.8.53 X X X -zerofrom@0.1.8 X -zerofrom-derive@0.1.7 X -zeroize@1.9.0 X X -zerotrie@0.2.4 X -zerovec@0.11.6 X -zerovec-derive@0.11.3 X -zlib-rs@0.6.5 X -zmij@1.0.21 X -zstd@0.13.3 X -zstd-safe@7.2.4 X X -zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/kms/aws/LICENSE b/crates/kms/aws/LICENSE deleted file mode 120000 index 5853aaea53..0000000000 --- a/crates/kms/aws/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../../LICENSE \ No newline at end of file diff --git a/crates/kms/aws/NOTICE b/crates/kms/aws/NOTICE deleted file mode 120000 index 295f6bdb3a..0000000000 --- a/crates/kms/aws/NOTICE +++ /dev/null @@ -1 +0,0 @@ -../../../NOTICE \ No newline at end of file diff --git a/crates/kms/aws/public-api.txt b/crates/kms/aws/public-api.txt deleted file mode 100644 index 7025896db2..0000000000 --- a/crates/kms/aws/public-api.txt +++ /dev/null @@ -1,23 +0,0 @@ -pub mod iceberg_kms_aws -pub struct iceberg_kms_aws::AwsKmsClientFactory -impl iceberg_kms_aws::AwsKmsClientFactory -pub fn iceberg_kms_aws::AwsKmsClientFactory::new() -> Self -pub fn iceberg_kms_aws::AwsKmsClientFactory::with_sdk_config(self, sdk_config: aws_types::sdk_config::SdkConfig) -> Self -impl core::clone::Clone for iceberg_kms_aws::AwsKmsClientFactory -pub fn iceberg_kms_aws::AwsKmsClientFactory::clone(&self) -> iceberg_kms_aws::AwsKmsClientFactory -impl core::default::Default for iceberg_kms_aws::AwsKmsClientFactory -pub fn iceberg_kms_aws::AwsKmsClientFactory::default() -> iceberg_kms_aws::AwsKmsClientFactory -impl core::fmt::Debug for iceberg_kms_aws::AwsKmsClientFactory -pub fn iceberg_kms_aws::AwsKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl iceberg::encryption::kms::factory::KmsClientFactory for iceberg_kms_aws::AwsKmsClientFactory -pub fn iceberg_kms_aws::AwsKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub const iceberg_kms_aws::AWS_ACCESS_KEY_ID: &str -pub const iceberg_kms_aws::AWS_PROFILE_NAME: &str -pub const iceberg_kms_aws::AWS_REGION_NAME: &str -pub const iceberg_kms_aws::AWS_SECRET_ACCESS_KEY: &str -pub const iceberg_kms_aws::AWS_SESSION_TOKEN: &str -pub const iceberg_kms_aws::KMS_DATA_KEY_SPEC: &str -pub const iceberg_kms_aws::KMS_DATA_KEY_SPEC_DEFAULT: &str -pub const iceberg_kms_aws::KMS_ENCRYPTION_ALGORITHM_SPEC: &str -pub const iceberg_kms_aws::KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str -pub const iceberg_kms_aws::KMS_ENDPOINT: &str diff --git a/crates/kms/public-api.txt b/crates/kms/public-api.txt new file mode 100644 index 0000000000..ef54d87661 --- /dev/null +++ b/crates/kms/public-api.txt @@ -0,0 +1,24 @@ +pub mod iceberg_kms +pub mod iceberg_kms::aws +pub struct iceberg_kms::aws::AwsKmsClientFactory +impl iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::new() -> Self +pub fn iceberg_kms::aws::AwsKmsClientFactory::with_sdk_config(self, sdk_config: aws_types::sdk_config::SdkConfig) -> Self +impl core::clone::Clone for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::clone(&self) -> iceberg_kms::aws::AwsKmsClientFactory +impl core::default::Default for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::default() -> iceberg_kms::aws::AwsKmsClientFactory +impl core::fmt::Debug for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::kms::factory::KmsClientFactory for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub const iceberg_kms::aws::AWS_ACCESS_KEY_ID: &str +pub const iceberg_kms::aws::AWS_PROFILE_NAME: &str +pub const iceberg_kms::aws::AWS_REGION_NAME: &str +pub const iceberg_kms::aws::AWS_SECRET_ACCESS_KEY: &str +pub const iceberg_kms::aws::AWS_SESSION_TOKEN: &str +pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC: &str +pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC_DEFAULT: &str +pub const iceberg_kms::aws::KMS_ENCRYPTION_ALGORITHM_SPEC: &str +pub const iceberg_kms::aws::KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str +pub const iceberg_kms::aws::KMS_ENDPOINT: &str diff --git a/crates/kms/aws/src/client.rs b/crates/kms/src/aws/client.rs similarity index 100% rename from crates/kms/aws/src/client.rs rename to crates/kms/src/aws/client.rs diff --git a/crates/kms/aws/src/config.rs b/crates/kms/src/aws/config.rs similarity index 100% rename from crates/kms/aws/src/config.rs rename to crates/kms/src/aws/config.rs diff --git a/crates/kms/aws/src/factory.rs b/crates/kms/src/aws/factory.rs similarity index 98% rename from crates/kms/aws/src/factory.rs rename to crates/kms/src/aws/factory.rs index 32c5c77bdd..9bbd338d18 100644 --- a/crates/kms/aws/src/factory.rs +++ b/crates/kms/src/aws/factory.rs @@ -26,8 +26,8 @@ use aws_sdk_kms::config::{Builder as KmsConfigBuilder, Credentials, Region}; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::{Error, ErrorKind, Result}; -use crate::client::AwsKeyManagementClient; -use crate::config::{ +use super::client::AwsKeyManagementClient; +use super::config::{ AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, AwsKmsConfig, }; @@ -148,7 +148,7 @@ mod tests { use iceberg::encryption::kms::KmsClientFactory; use super::*; - use crate::KMS_ENDPOINT; + use crate::aws::KMS_ENDPOINT; #[tokio::test] async fn test_create_client_with_injected_sdk_config() { diff --git a/crates/kms/aws/src/lib.rs b/crates/kms/src/aws/mod.rs similarity index 92% rename from crates/kms/aws/src/lib.rs rename to crates/kms/src/aws/mod.rs index 69246bc1d2..9782e7ac2b 100644 --- a/crates/kms/aws/src/lib.rs +++ b/crates/kms/src/aws/mod.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -//! AWS Key Management Service integration for Apache Iceberg client-side encryption. +//! AWS Key Management Service integration. //! -//! This crate provides [`AwsKmsClientFactory`], which creates AWS-backed -//! implementations of Iceberg's +//! This module provides [`AwsKmsClientFactory`](crate::aws::AwsKmsClientFactory), +//! which creates AWS-backed implementations of Iceberg's //! [`KeyManagementClient`](iceberg::encryption::KeyManagementClient). The factory can //! be supplied to any Iceberg catalog builder through //! [`CatalogBuilder::with_kms_client_factory`](iceberg::CatalogBuilder::with_kms_client_factory). @@ -59,7 +59,7 @@ //! //! use iceberg::CatalogBuilder; //! use iceberg_catalog_rest::RestCatalogBuilder; -//! use iceberg_kms_aws::AwsKmsClientFactory; +//! use iceberg_kms::aws::AwsKmsClientFactory; //! //! # async fn example() -> iceberg::Result<()> { //! let catalog = RestCatalogBuilder::default() @@ -71,8 +71,6 @@ //! # } //! ``` -#![deny(missing_docs)] - mod client; mod config; mod factory; diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs new file mode 100644 index 0000000000..33b06c53c4 --- /dev/null +++ b/crates/kms/src/lib.rs @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key management service integrations for Apache Iceberg client-side encryption. +//! +//! Provider implementations are enabled through crate features. Enable `aws` +//! to use the `aws` module. + +#![deny(missing_docs)] + +/// AWS Key Management Service integration. +#[cfg(feature = "aws")] +pub mod aws; From 1833d1ecb95c8108cc9dde4befaf513903190e0d Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Wed, 7 Oct 2026 15:47:41 +0100 Subject: [PATCH 6/8] refactor(kms): address review feedback on the AWS KMS client - Follow Iceberg Java for configuration: honour `client.region` (with `region_name` as a lower-precedence alias) and parse all catalog properties through `iceberg_property_macro::Properties`. Glue-style property constants are now crate-private. - Reject `client.assume-role.arn` until assume-role support lands in a follow-up, instead of silently using different credentials than the catalog's S3 file IO. - Report AWS service errors by their modeled error, and transport errors by their full cause chain with the Encrypt plaintext redacted. Never attach the raw HTTP response, which can contain plaintext keys. - Validate `kms.endpoint` as an http(s) URI with a host when the client is created, and reject GenerateDataKey keys that do not match `kms.data-key-spec` and empty ciphertext blobs. - Skip SDK catalog properties entirely when an `SdkConfig` is injected. - Raise aws-sdk-kms to 1.123, which no longer pulls in h2 0.3 through test-util, and drop the RUSTSEC-2026-0258 audit exception. Set the aws-config floor to 1.9, the minimum compatible with aws-sdk-kms 1.123. - Move the AWS dev-dependencies to the workspace, drop the unused iceberg-catalog-rest dev-dependency, and use the memory catalog in the module example. - Isolate tests from the host AWS environment, cover the factory wiring of every property, and remove redundant and SDK-wording-dependent assertions. --- .cargo/audit.toml | 14 - Cargo.lock | 185 ++++-------- Cargo.toml | 7 +- README.md | 12 +- crates/kms/Cargo.toml | 8 +- crates/kms/DEPENDENCIES.rust.tsv | 5 - crates/kms/public-api.txt | 5 - crates/kms/src/aws/client.rs | 473 ++++++++++++++++++++++++++----- crates/kms/src/aws/config.rs | 173 ++++++++++- crates/kms/src/aws/factory.rs | 277 ++++++++++++------ crates/kms/src/aws/mod.rs | 79 ++++-- crates/kms/src/lib.rs | 1 - 12 files changed, 886 insertions(+), 353 deletions(-) diff --git a/.cargo/audit.toml b/.cargo/audit.toml index cd60d23475..176d43abf6 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -37,18 +37,4 @@ ignore = [ # once the transitive dependency set permits quick-xml >=0.41.0. "RUSTSEC-2026-0194", "RUSTSEC-2026-0195", - # `h2` 0.3.27 accepts an unbounded number of empty DATA frames (low-severity - # DoS). - # - # It is present only through `aws-sdk-kms/test-util` -> - # `aws-smithy-runtime/legacy-test-util`, which is required by the official - # `aws-smithy-mocks` test framework, and is absent from normal/build - # dependencies. - # - # Per SECURITY-THREAT-MODEL.md, malformed-input DoS is hardening rather than - # an Iceberg Rust trust-boundary vulnerability. - # - # Remove after the fix in - # https://github.com/smithy-lang/smithy-rs/pull/4827 reaches an AWS SDK release. - "RUSTSEC-2026-0258", ] diff --git a/Cargo.lock b/Cargo.lock index 6359342cd2..a86e4f2121 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,7 +544,7 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-json 0.63.1", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -600,14 +600,14 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.4" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25b43ad47adc2517efe3d706559d94b97e50e80e0321b3cadbc9f77cee88adcd" +checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -616,7 +616,7 @@ dependencies = [ "bytes-utils", "fastrand", "http 1.5.0", - "http-body 1.0.1", + "http-body", "percent-encoding", "pin-project-lite", "tracing", @@ -650,15 +650,15 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.120.0" +version = "1.123.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b0fe38fee2ba5b6cd24d32d08365b314ae1adea123649e061a7eb6300b6f5b" +checksum = "0bbe8833a7ad3b970f2a6a856939c9cb171942dcdfd2ab65f2db3a8baaf73cc1" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-json 0.63.1", "aws-smithy-observability 0.3.0", "aws-smithy-runtime", @@ -668,7 +668,6 @@ dependencies = [ "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -701,15 +700,15 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.111.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90" +checksum = "12be2f9c8eef7f5fc919c96d538e629698469a02b4cb75408b26b1bd984ebe79" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-json 0.63.1", "aws-smithy-observability 0.3.0", "aws-smithy-runtime", @@ -719,7 +718,6 @@ dependencies = [ "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -727,15 +725,15 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.113.0" +version = "1.116.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a" +checksum = "d7645db8724ea3b82fdccfb67e1b0f637c9d8ab0e7ef29d84884d8d8d73f805d" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-json 0.63.1", "aws-smithy-observability 0.3.0", "aws-smithy-runtime", @@ -745,7 +743,6 @@ dependencies = [ "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -753,15 +750,15 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.116.0" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "035c7392e751d20bf1060d625d023587eb14099646110a9cff7c697bfe4b253a" +checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-json 0.63.1", "aws-smithy-observability 0.3.0", "aws-smithy-query", @@ -772,7 +769,6 @@ dependencies = [ "aws-smithy-xml", "aws-types", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -780,19 +776,18 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31d955e76ff96acd555bf06fa0fa6d5bf9335fa84ae7c64481b20ae61d231f70" +checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" dependencies = [ "aws-credential-types", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "form_urlencoded", "hex", "hmac 0.13.0", - "http 0.2.12", "http 1.5.0", "percent-encoding", "sha2 0.11.0", @@ -824,7 +819,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", "percent-encoding", "pin-project-lite", @@ -834,9 +829,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.0" +version = "0.64.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -845,7 +840,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", "percent-encoding", "pin-project-lite", @@ -855,23 +850,19 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd25384a4e437aa8d8f339afad4b69e786b936a7cb10db668a7aaf66717b1a8" +checksum = "51c89cc3f1f281d659a67a519a1b5c6d445b5ce09fa7e5aee40c2c2707e9509d" dependencies = [ "aws-smithy-async", "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "h2 0.3.27", - "h2 0.4.16", - "http 0.2.12", + "h2", "http 1.5.0", - "http-body 0.4.6", - "http-body 1.0.1", - "hyper 0.14.32", - "hyper 1.10.1", + "http-body", + "hyper", "hyper-rustls", "hyper-util", "indexmap 2.14.0", @@ -973,12 +964,12 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.14.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3296253d3a91b3f938a3f2bcce4daebadcb4aa4228153fcec90f9d23532b4484" +checksum = "d6e302ac1d88b99652489df31abdec6ac42a2ab2ac3982ad0ac49f64dfaf28ba" dependencies = [ "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-http-client", "aws-smithy-observability 0.3.0", "aws-smithy-runtime-api", @@ -986,29 +977,25 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", - "http-body 1.0.1", + "http-body", "http-body-util", "pin-project-lite", "pin-utils", "tokio", "tracing", - "tracing-subscriber", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.16.2" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d881a7b7ad179fd6611680c9de89f716fb00ab40299a9a7b8c6913e8f7511a8" +checksum = "c0730c16f91124c6a2abb4932c77e299288b3dd9f967ea2e9ec48cc6731e87a4" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", - "http 0.2.12", "http 1.5.0", "pin-project-lite", "tokio", @@ -1051,18 +1038,16 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.4" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b791f3ac597193fe1d08b82366986eb1f5bc31f2ac6c194c0855276116c76cd" +checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", - "http-body 1.0.1", + "http-body", "http-body-util", "itoa", "num-integer", @@ -1541,7 +1526,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -2507,25 +2492,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.16" @@ -2705,17 +2671,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -2735,7 +2690,7 @@ dependencies = [ "bytes", "futures-core", "http 1.5.0", - "http-body 1.0.1", + "http-body", "pin-project-lite", ] @@ -2766,30 +2721,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.10.1" @@ -2800,9 +2731,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2 0.4.16", + "h2", "http 1.5.0", - "http-body 1.0.1", + "http-body", "httparse", "httpdate", "itoa", @@ -2819,7 +2750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.5.0", - "hyper 1.10.1", + "hyper", "hyper-util", "rustls", "rustls-native-certs", @@ -2839,8 +2770,8 @@ dependencies = [ "futures-channel", "futures-util", "http 1.5.0", - "http-body 1.0.1", - "hyper 1.10.1", + "http-body", + "hyper", "ipnet", "libc", "percent-encoding", @@ -3102,9 +3033,11 @@ dependencies = [ "async-trait", "aws-config", "aws-sdk-kms", + "aws-smithy-http-client", "aws-smithy-mocks", + "base64 0.22.1", + "http 1.5.0", "iceberg", - "iceberg-catalog-rest", "iceberg-property-macro", "tokio", ] @@ -3822,9 +3755,9 @@ dependencies = [ "colored", "futures-core", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", - "hyper 1.10.1", + "hyper", "hyper-util", "log", "pin-project-lite", @@ -4180,7 +4113,7 @@ dependencies = [ "bytes", "futures", "http 1.5.0", - "http-body 1.0.1", + "http-body", "opendal-core", "reqwest 0.13.4", ] @@ -5265,9 +5198,9 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", - "hyper 1.10.1", + "hyper", "hyper-util", "js-sys", "log", @@ -5300,9 +5233,9 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", - "hyper 1.10.1", + "hyper", "hyper-rustls", "hyper-util", "js-sys", @@ -6430,7 +6363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -6713,7 +6646,7 @@ dependencies = [ "bytes", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "pin-project-lite", "tower", "tower-layer", @@ -7306,7 +7239,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -7629,7 +7562,7 @@ dependencies = [ "crc32fast", "futures", "http 1.5.0", - "hyper 1.10.1", + "hyper", "lazy_static", "more-asserts", "rand 0.10.2", diff --git a/Cargo.toml b/Cargo.toml index 29a8eec366..b3e0278151 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,12 +60,12 @@ arrow-select = "59.2" arrow-string = "59.2" as-any = "0.3.2" async-trait = "0.1.89" -aws-config = "1.12" +aws-config = "1.9" aws-sdk-glue = { version = "1.85", default-features = false, features = [ "default-https-client", "rt-tokio", ] } -aws-sdk-kms = { version = "1.118", default-features = false, features = [ +aws-sdk-kms = { version = "1.123", default-features = false, features = [ "default-https-client", "rt-tokio", ] } @@ -73,6 +73,8 @@ aws-sdk-s3tables = { version = "1.28", default-features = false, features = [ "default-https-client", "rt-tokio", ] } +aws-smithy-http-client = "1.5" +aws-smithy-mocks = "0.3" backon = "1.5.1" base64 = "0.22.1" bimap = "0.6" @@ -99,7 +101,6 @@ iceberg-catalog-hms = { version = "0.10.0", path = "./crates/catalog/hms" } iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } -iceberg-kms = { version = "0.10.0", path = "./crates/kms" } iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } itertools = "0.13" diff --git a/README.md b/README.md index 14d63b3763..ca2a96edad 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,6 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg release docs]: https://docs.rs/iceberg [iceberg dev docs]: https://rust.iceberg.apache.org/api/iceberg/ -[iceberg-kms]: crates/kms/README.md -[iceberg-kms image]: https://img.shields.io/crates/v/iceberg-kms.svg -[iceberg-kms link]: https://crates.io/crates/iceberg-kms -[iceberg-kms release docs]: https://docs.rs/iceberg-kms -[iceberg-kms dev docs]: https://rust.iceberg.apache.org/api/iceberg_kms/ - [iceberg-catalog-glue]: crates/catalog/glue/README.md [iceberg-catalog-glue image]: https://img.shields.io/crates/v/iceberg-catalog-glue.svg [iceberg-catalog-glue link]: https://crates.io/crates/iceberg-catalog-glue @@ -102,6 +96,12 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg-cache-moka release docs]: https://docs.rs/iceberg-cache-moka [iceberg-cache-moka dev docs]: https://rust.iceberg.apache.org/api/iceberg_cache_moka/ +[iceberg-kms]: crates/kms/README.md +[iceberg-kms image]: https://img.shields.io/crates/v/iceberg-kms.svg +[iceberg-kms link]: https://crates.io/crates/iceberg-kms +[iceberg-kms release docs]: https://docs.rs/iceberg-kms +[iceberg-kms dev docs]: https://rust.iceberg.apache.org/api/iceberg_kms/ + ## Iceberg Rust Implementation Status The features that Iceberg Rust currently supports can be found [here](https://iceberg.apache.org/status/). diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml index 38fbf56db6..6c91400480 100644 --- a/crates/kms/Cargo.toml +++ b/crates/kms/Cargo.toml @@ -38,6 +38,8 @@ aws = [ "dep:async-trait", "dep:aws-config", "dep:aws-sdk-kms", + "dep:base64", + "dep:http", "dep:iceberg-property-macro", ] default = [] @@ -46,13 +48,15 @@ default = [] async-trait = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } aws-sdk-kms = { workspace = true, optional = true } +base64 = { workspace = true, optional = true } +http = { workspace = true, optional = true } iceberg = { workspace = true } iceberg-property-macro = { workspace = true, optional = true } [dev-dependencies] aws-sdk-kms = { workspace = true, features = ["test-util"] } -aws-smithy-mocks = "0.3" -iceberg-catalog-rest = { workspace = true } +aws-smithy-http-client = { workspace = true, features = ["test-util"] } +aws-smithy-mocks = { workspace = true } tokio = { workspace = true, features = ["macros"] } [lints] diff --git a/crates/kms/DEPENDENCIES.rust.tsv b/crates/kms/DEPENDENCIES.rust.tsv index 1c4d66ae92..da201d49dd 100644 --- a/crates/kms/DEPENDENCIES.rust.tsv +++ b/crates/kms/DEPENDENCIES.rust.tsv @@ -108,13 +108,11 @@ http@1.5.0 X X http-body@1.0.1 X http-body-util@0.1.3 X httparse@1.10.1 X X -httpdate@1.0.3 X X hyper@1.10.1 X hyper-util@0.1.20 X iana-time-zone@0.1.65 X X iana-time-zone-haiku@0.1.2 X X iceberg@0.10.1 X -iceberg-catalog-rest@0.10.1 X iceberg-kms@0.10.1 X iceberg-property-macro@0.10.1 X iceberg_test_utils@0.10.1 X @@ -149,7 +147,6 @@ litemap@0.8.2 X lock_api@0.4.14 X X log@0.4.33 X X lz4_flex@0.14.0 X -matchers@0.2.0 X memchr@2.8.2 X X miniz_oxide@0.8.9 X X X mio@1.2.1 X @@ -242,7 +239,6 @@ tracing@0.1.44 X tracing-attributes@0.1.31 X tracing-core@0.1.36 X tracing-log@0.2.0 X -tracing-serde@0.2.0 X tracing-subscriber@0.3.23 X try-lock@0.2.5 X twox-hash@2.1.2 X @@ -267,7 +263,6 @@ wasm-bindgen-futures@0.4.76 X X wasm-bindgen-macro@0.2.126 X X wasm-bindgen-macro-support@0.2.126 X X wasm-bindgen-shared@0.2.126 X X -wasm-streams@0.4.2 X X web-sys@0.3.103 X X windows-core@0.62.2 X X windows-implement@0.60.2 X X diff --git a/crates/kms/public-api.txt b/crates/kms/public-api.txt index ef54d87661..34c635f643 100644 --- a/crates/kms/public-api.txt +++ b/crates/kms/public-api.txt @@ -12,11 +12,6 @@ impl core::fmt::Debug for iceberg_kms::aws::AwsKmsClientFactory pub fn iceberg_kms::aws::AwsKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg::encryption::kms::factory::KmsClientFactory for iceberg_kms::aws::AwsKmsClientFactory pub fn iceberg_kms::aws::AwsKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait -pub const iceberg_kms::aws::AWS_ACCESS_KEY_ID: &str -pub const iceberg_kms::aws::AWS_PROFILE_NAME: &str -pub const iceberg_kms::aws::AWS_REGION_NAME: &str -pub const iceberg_kms::aws::AWS_SECRET_ACCESS_KEY: &str -pub const iceberg_kms::aws::AWS_SESSION_TOKEN: &str pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC: &str pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC_DEFAULT: &str pub const iceberg_kms::aws::KMS_ENCRYPTION_ALGORITHM_SPEC: &str diff --git a/crates/kms/src/aws/client.rs b/crates/kms/src/aws/client.rs index 3abd93cd3e..47ce97408e 100644 --- a/crates/kms/src/aws/client.rs +++ b/crates/kms/src/aws/client.rs @@ -18,21 +18,24 @@ use std::fmt; use async_trait::async_trait; -use aws_sdk_kms::Client; +use aws_sdk_kms::error::{ProvideErrorMetadata, SdkError}; use aws_sdk_kms::primitives::Blob; use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; -use iceberg::encryption::{GeneratedKey, KeyManagementClient, SensitiveBytes}; +use base64::Engine as _; +use base64::prelude::BASE64_STANDARD; +use iceberg::encryption::SensitiveBytes; +use iceberg::encryption::kms::{GeneratedKey, KeyManagementClient}; use iceberg::{Error, ErrorKind, Result}; pub(crate) struct AwsKeyManagementClient { - client: Client, + client: aws_sdk_kms::Client, encryption_algorithm: EncryptionAlgorithmSpec, data_key_spec: DataKeySpec, } impl AwsKeyManagementClient { pub(crate) fn new( - client: Client, + client: aws_sdk_kms::Client, encryption_algorithm: EncryptionAlgorithmSpec, data_key_spec: DataKeySpec, ) -> Self { @@ -43,12 +46,80 @@ impl AwsKeyManagementClient { } } + /// Converts a failed KMS request into an Iceberg error. + /// + /// `plaintext` is the key sent in the request, if any. The SDK can describe + /// the request, including its body, in transport errors (for example when + /// an endpoint URL cannot be applied), so its base64 encoding is redacted + /// from the message. The raw HTTP response, which holds the plaintext key + /// when a Decrypt or GenerateDataKey response fails to parse, is dropped. + fn request_failed( + operation: &str, + wrapping_key_id: &str, + plaintext: Option<&[u8]>, + error: SdkError, + ) -> Error + where + E: ProvideErrorMetadata + std::error::Error + Send + Sync + 'static, + R: fmt::Debug, + { + let message = |cause: String| { + format!("AWS KMS {operation} failed for key '{wrapping_key_id}': {cause}") + }; + match error { + SdkError::ServiceError(context) => { + let service_error = context.into_err(); + let description = service_error.to_string(); + let cause = match service_error.message() { + Some(aws_message) if !description.contains(aws_message) => { + format!("{description}: {aws_message}") + } + _ => description, + }; + Error::new(ErrorKind::Unexpected, message(cause)).with_source(service_error) + } + error => { + let mut causes = + std::iter::successors(Some(&error as &dyn std::error::Error), |error| { + (*error).source() + }) + .map(ToString::to_string) + .collect::>(); + causes.dedup(); + let cause = causes.join(": "); + let cause = match plaintext.filter(|plaintext| !plaintext.is_empty()) { + Some(plaintext) => { + cause.replace(&BASE64_STANDARD.encode(plaintext), "") + } + None => cause, + }; + // The cause chain is not attached as a source, since only the + // message is redacted. + Error::new(ErrorKind::Unexpected, message(cause)) + } + } + } + fn missing_response_field(operation: &str, field: &str) -> Error { Error::new( ErrorKind::Unexpected, format!("AWS KMS {operation} response did not contain {field}"), ) } + + fn ciphertext_blob(operation: &str, blob: Option) -> Result> { + blob.map(Blob::into_inner) + .filter(|ciphertext| !ciphertext.is_empty()) + .ok_or_else(|| Self::missing_response_field(operation, "ciphertext_blob")) + } + + fn generated_key_length(&self) -> Option { + match self.data_key_spec { + DataKeySpec::Aes128 => Some(16), + DataKeySpec::Aes256 => Some(32), + _ => None, + } + } } impl fmt::Debug for AwsKeyManagementClient { @@ -72,17 +143,10 @@ impl KeyManagementClient for AwsKeyManagementClient { .send() .await .map_err(|source| { - Error::new( - ErrorKind::Unexpected, - format!("AWS KMS Encrypt failed for key '{wrapping_key_id}'"), - ) - .with_source(source) + Self::request_failed("Encrypt", wrapping_key_id, Some(key), source) })?; - response - .ciphertext_blob - .map(Blob::into_inner) - .ok_or_else(|| Self::missing_response_field("Encrypt", "ciphertext_blob")) + Self::ciphertext_blob("Encrypt", response.ciphertext_blob) } async fn unwrap_key( @@ -98,13 +162,7 @@ impl KeyManagementClient for AwsKeyManagementClient { .ciphertext_blob(Blob::new(wrapped_key)) .send() .await - .map_err(|source| { - Error::new( - ErrorKind::Unexpected, - format!("AWS KMS Decrypt failed for key '{wrapping_key_id}'"), - ) - .with_source(source) - })?; + .map_err(|source| Self::request_failed("Decrypt", wrapping_key_id, None, source))?; response .plaintext @@ -125,21 +183,26 @@ impl KeyManagementClient for AwsKeyManagementClient { .send() .await .map_err(|source| { - Error::new( - ErrorKind::Unexpected, - format!("AWS KMS GenerateDataKey failed for key '{wrapping_key_id}'"), - ) - .with_source(source) + Self::request_failed("GenerateDataKey", wrapping_key_id, None, source) })?; let plaintext = response .plaintext .map(|blob| SensitiveBytes::new(blob.into_inner())) .ok_or_else(|| Self::missing_response_field("GenerateDataKey", "plaintext"))?; - let wrapped_key = response - .ciphertext_blob - .map(Blob::into_inner) - .ok_or_else(|| Self::missing_response_field("GenerateDataKey", "ciphertext_blob"))?; + if let Some(expected) = self.generated_key_length() + && plaintext.len() != expected + { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "AWS KMS GenerateDataKey returned a {}-byte key for key spec {}, expected {expected} bytes", + plaintext.len(), + self.data_key_spec.as_str() + ), + )); + } + let wrapped_key = Self::ciphertext_blob("GenerateDataKey", response.ciphertext_blob)?; Ok(GeneratedKey::new(plaintext, wrapped_key)) } @@ -147,14 +210,13 @@ impl KeyManagementClient for AwsKeyManagementClient { #[cfg(test)] mod tests { - use std::error::Error as StdError; use std::sync::{Arc, Mutex}; - use aws_sdk_kms::Client; - use aws_sdk_kms::operation::decrypt::DecryptOutput; + use aws_sdk_kms::error::{ConnectorError, ErrorMetadata}; + use aws_sdk_kms::operation::decrypt::{DecryptError, DecryptOutput}; use aws_sdk_kms::operation::encrypt::{EncryptError, EncryptOutput}; - use aws_sdk_kms::operation::generate_data_key::GenerateDataKeyOutput; - use aws_sdk_kms::types::error::NotFoundException; + use aws_sdk_kms::operation::generate_data_key::{GenerateDataKeyError, GenerateDataKeyOutput}; + use aws_sdk_kms::types::error::{DisabledException, NotFoundException}; use aws_smithy_mocks::{RuleMode, mock, mock_client}; use iceberg::encryption::{EncryptionManager, StandardKeyMetadata}; @@ -166,7 +228,7 @@ mod tests { async fn test_wrap_key() { let plaintext = b"0123456789abcdef"; let ciphertext = b"wrapped-key"; - let rule = mock!(Client::encrypt) + let rule = mock!(aws_sdk_kms::Client::encrypt) .match_requests(|request| { request.key_id() == Some(KEY_ID) && request.encryption_algorithm() @@ -189,7 +251,7 @@ mod tests { async fn test_unwrap_key() { let plaintext = b"0123456789abcdef"; let ciphertext = b"wrapped-key"; - let rule = mock!(Client::decrypt) + let rule = mock!(aws_sdk_kms::Client::decrypt) .match_requests(|request| { request.key_id() == Some(KEY_ID) && request.encryption_algorithm() @@ -215,7 +277,7 @@ mod tests { async fn test_generate_key() { let plaintext = b"0123456789abcdef0123456789abcdef"; let ciphertext = b"generated-wrapped-key"; - let rule = mock!(Client::generate_data_key) + let rule = mock!(aws_sdk_kms::Client::generate_data_key) .match_requests(|request| { request.key_id() == Some(KEY_ID) && request.key_spec() == Some(&DataKeySpec::Aes256) }) @@ -260,7 +322,7 @@ mod tests { async fn test_asymmetric_manager_roundtrip_uses_encrypt_instead_of_generate_data_key() { let plaintext_kek = Arc::new(Mutex::new(Vec::new())); let plaintext_kek_from_encrypt = Arc::clone(&plaintext_kek); - let encrypt_rule = mock!(Client::encrypt) + let encrypt_rule = mock!(aws_sdk_kms::Client::encrypt) .match_requests(move |request| { if request.key_id() != Some(KEY_ID) || request.encryption_algorithm() @@ -282,11 +344,19 @@ mod tests { .build() }); let plaintext_kek_for_decrypt = Arc::clone(&plaintext_kek); - let decrypt_rule = mock!(Client::decrypt).then_output(move || { - DecryptOutput::builder() - .plaintext(Blob::new(plaintext_kek_for_decrypt.lock().unwrap().clone())) - .build() - }); + let decrypt_rule = mock!(aws_sdk_kms::Client::decrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::RsaesOaepSha256) + && request.ciphertext_blob().map(AsRef::as_ref) + == Some(b"wrapped-asymmetric-kek".as_slice()) + }) + .then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext_kek_for_decrypt.lock().unwrap().clone())) + .build() + }); let client = mock_client!(aws_sdk_kms, &[&encrypt_rule, &decrypt_rule]); let kms: Arc = Arc::new(AwsKeyManagementClient::new( client, @@ -322,29 +392,23 @@ mod tests { } #[tokio::test] - async fn test_aws_service_error_is_preserved_as_source() { - let rule = mock!(Client::encrypt).then_error(|| { - EncryptError::NotFoundException( - NotFoundException::builder() - .message("test key not found") + async fn test_aws_service_error_messages_name_operation_and_key() { + let not_found = || { + NotFoundException::builder() + .message("test key not found") + .build() + }; + let encrypt_rule = mock!(aws_sdk_kms::Client::encrypt) + .then_error(move || EncryptError::NotFoundException(not_found())); + let decrypt_rule = mock!(aws_sdk_kms::Client::decrypt).then_error(|| { + DecryptError::DisabledException( + DisabledException::builder() + .message("test key disabled") .build(), ) }); - let client = mock_client!(aws_sdk_kms, &[&rule]); - let kms = test_client(client); - - let error = kms.wrap_key(b"key", KEY_ID).await.unwrap_err(); - - assert_eq!(error.kind(), ErrorKind::Unexpected); - assert!(error.source().is_some()); - } - - #[tokio::test] - async fn test_missing_response_fields() { - let encrypt_rule = mock!(Client::encrypt).then_output(|| EncryptOutput::builder().build()); - let decrypt_rule = mock!(Client::decrypt).then_output(|| DecryptOutput::builder().build()); - let generate_rule = mock!(Client::generate_data_key) - .then_output(|| GenerateDataKeyOutput::builder().build()); + let generate_rule = mock!(aws_sdk_kms::Client::generate_data_key) + .then_error(move || GenerateDataKeyError::NotFoundException(not_found())); let client = mock_client!(aws_sdk_kms, RuleMode::Sequential, &[ &encrypt_rule, &decrypt_rule, @@ -354,33 +418,292 @@ mod tests { let error = kms.wrap_key(b"key", KEY_ID).await.unwrap_err(); assert_eq!(error.kind(), ErrorKind::Unexpected); - assert!(error.message().contains("ciphertext_blob")); + assert!(std::error::Error::source(&error).is_some()); + assert_eq!( + error.message(), + format!( + "AWS KMS Encrypt failed for key '{KEY_ID}': NotFoundException: test key not found" + ) + ); let error = kms.unwrap_key(b"wrapped", KEY_ID).await.unwrap_err(); assert_eq!(error.kind(), ErrorKind::Unexpected); - assert!(error.message().contains("plaintext")); + assert!( + error + .message() + .starts_with(&format!("AWS KMS Decrypt failed for key '{KEY_ID}': ")), + "{error}" + ); + assert!(error.message().contains("test key disabled"), "{error}"); let error = kms.generate_key(KEY_ID).await.err().unwrap(); assert_eq!(error.kind(), ErrorKind::Unexpected); - assert!(error.message().contains("plaintext")); + assert!( + error.message().starts_with(&format!( + "AWS KMS GenerateDataKey failed for key '{KEY_ID}': " + )), + "{error}" + ); + assert!(error.message().contains("test key not found"), "{error}"); + } + + #[test] + fn test_request_failure_describes_transport_errors() { + let error = SdkError::::dispatch_failure(ConnectorError::other( + "an error occurred while loading credentials".into(), + None, + )); + + let error = AwsKeyManagementClient::request_failed("Decrypt", KEY_ID, None, error); + + // Nested transport errors are part of the message. + assert!( + error + .message() + .ends_with(": an error occurred while loading credentials"), + "{error}" + ); + } + + #[test] + fn test_request_failure_redacts_plaintext_key() { + let plaintext = b"0123456789abcdef"; + let encoded = BASE64_STANDARD.encode(plaintext); + let error = SdkError::::dispatch_failure(ConnectorError::other( + format!(r#"failed to apply endpoint to request {{"Plaintext":"{encoded}"}}"#).into(), + None, + )); + + let error = + AwsKeyManagementClient::request_failed("Encrypt", KEY_ID, Some(plaintext), error); + + assert!(error.message().contains(""), "{error}"); + for output in [ + error.to_string(), + format!("{error:?}"), + format!("{error:#?}"), + ] { + assert!(!output.contains(&encoded), "{output}"); + } + } + + #[test] + fn test_request_failure_includes_unmodeled_error_code_and_message() { + let error = SdkError::::service_error( + DecryptError::generic( + ErrorMetadata::builder() + .code("AccessDeniedException") + .message("User is not authorized to perform: kms:Decrypt") + .build(), + ), + String::new(), + ); + + let error = AwsKeyManagementClient::request_failed("Decrypt", KEY_ID, None, error); + + assert!(error.message().contains("AccessDeniedException"), "{error}"); + assert!( + error + .message() + .ends_with(": User is not authorized to perform: kms:Decrypt"), + "{error}" + ); + } + + #[test] + fn test_request_failure_drops_raw_response() { + let error = SdkError::service_error( + DecryptError::NotFoundException(NotFoundException::builder().build()), + r#"{"Plaintext":"SECRET"}"#.to_string(), + ); + + let error = AwsKeyManagementClient::request_failed("Decrypt", KEY_ID, None, error); + + let debug = format!("{error:#?}"); + assert!(debug.contains("NotFoundException"), "{debug}"); + assert!(!debug.contains("SECRET"), "{debug}"); } #[tokio::test] - async fn test_generate_key_requires_wrapped_key() { - let rule = mock!(Client::generate_data_key).then_output(|| { - GenerateDataKeyOutput::builder() - .plaintext(Blob::new([0; 32])) + async fn test_reject_missing_or_empty_response_fields() { + let encrypt_missing = + mock!(aws_sdk_kms::Client::encrypt).then_output(|| EncryptOutput::builder().build()); + let encrypt_empty = mock!(aws_sdk_kms::Client::encrypt).then_output(|| { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(Vec::new())) .build() }); - let client = mock_client!(aws_sdk_kms, &[&rule]); + let decrypt_missing = + mock!(aws_sdk_kms::Client::decrypt).then_output(|| DecryptOutput::builder().build()); + let generate_missing_plaintext = + mock!(aws_sdk_kms::Client::generate_data_key).then_output(|| { + GenerateDataKeyOutput::builder() + .ciphertext_blob(Blob::new(b"wrapped")) + .build() + }); + let generate_missing_ciphertext = mock!(aws_sdk_kms::Client::generate_data_key) + .then_output(|| { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new([0; 32])) + .build() + }); + let generate_empty_ciphertext = + mock!(aws_sdk_kms::Client::generate_data_key).then_output(|| { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new([0; 32])) + .ciphertext_blob(Blob::new(Vec::new())) + .build() + }); + let client = mock_client!(aws_sdk_kms, RuleMode::Sequential, &[ + &encrypt_missing, + &encrypt_empty, + &decrypt_missing, + &generate_missing_plaintext, + &generate_missing_ciphertext, + &generate_empty_ciphertext + ]); let kms = test_client(client); - let error = kms.generate_key(KEY_ID).await.err().unwrap(); - assert_eq!(error.kind(), ErrorKind::Unexpected); - assert!(error.message().contains("ciphertext_blob")); + let errors = [ + kms.wrap_key(b"key", KEY_ID).await.unwrap_err(), + kms.wrap_key(b"key", KEY_ID).await.unwrap_err(), + kms.unwrap_key(b"wrapped", KEY_ID).await.unwrap_err(), + kms.generate_key(KEY_ID).await.err().unwrap(), + kms.generate_key(KEY_ID).await.err().unwrap(), + kms.generate_key(KEY_ID).await.err().unwrap(), + ]; + let expected = [ + "AWS KMS Encrypt response did not contain ciphertext_blob", + "AWS KMS Encrypt response did not contain ciphertext_blob", + "AWS KMS Decrypt response did not contain plaintext", + "AWS KMS GenerateDataKey response did not contain plaintext", + "AWS KMS GenerateDataKey response did not contain ciphertext_blob", + "AWS KMS GenerateDataKey response did not contain ciphertext_blob", + ]; + for (error, expected) in errors.iter().zip(expected) { + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert_eq!(error.message(), expected); + } + } + + #[tokio::test] + async fn test_reject_generated_key_not_matching_key_spec() { + for (data_key_spec, length, expected) in [ + ( + DataKeySpec::Aes256, + 24, + "a 24-byte key for key spec AES_256, expected 32 bytes", + ), + ( + DataKeySpec::Aes128, + 32, + "a 32-byte key for key spec AES_128, expected 16 bytes", + ), + ] { + let rule = mock!(aws_sdk_kms::Client::generate_data_key).then_output(move || { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new(vec![0; length])) + .ciphertext_blob(Blob::new(b"wrapped")) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::SymmetricDefault, + data_key_spec, + ); + + let error = kms.generate_key(KEY_ID).await.err().unwrap(); + + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert_eq!( + error.message(), + format!("AWS KMS GenerateDataKey returned {expected}") + ); + } + } + + #[test] + fn test_request_failure_does_not_repeat_aws_message() { + // Errors parsed from AWS responses carry the message in their metadata + // as well as in the modeled exception. + let error = SdkError::::service_error( + EncryptError::NotFoundException( + NotFoundException::builder() + .message("test key not found") + .meta( + ErrorMetadata::builder() + .code("NotFoundException") + .message("test key not found") + .build(), + ) + .build(), + ), + String::new(), + ); + + let error = AwsKeyManagementClient::request_failed("Encrypt", KEY_ID, None, error); + + assert_eq!( + error.message().matches("test key not found").count(), + 1, + "{error}" + ); + } + + #[tokio::test] + async fn test_symmetric_manager_roundtrip_uses_generate_data_key() { + let plaintext_kek = [7; 32]; + let wrapped_kek = b"wrapped-symmetric-kek"; + let generate_rule = mock!(aws_sdk_kms::Client::generate_data_key) + .match_requests(|request| request.key_id() == Some(KEY_ID)) + .then_output(move || { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new(plaintext_kek)) + .ciphertext_blob(Blob::new(wrapped_kek)) + .build() + }); + let decrypt_rule = mock!(aws_sdk_kms::Client::decrypt) + .match_requests(move |request| { + request.key_id() == Some(KEY_ID) + && request.ciphertext_blob().map(AsRef::as_ref) == Some(wrapped_kek.as_slice()) + }) + .then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext_kek)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&generate_rule, &decrypt_rule]); + let kms: Arc = Arc::new(test_client(client)); + let manager = EncryptionManager::builder() + .kms_client(Arc::clone(&kms)) + .table_key_id(KEY_ID) + .build(); + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let encrypted_key_id = manager + .encrypt_manifest_list_key_metadata(&key_metadata) + .await + .unwrap(); + let encryption_keys = manager.with_encryption_keys(Clone::clone); + let reader = EncryptionManager::builder() + .kms_client(kms) + .table_key_id(KEY_ID) + .encryption_keys(encryption_keys) + .build(); + let decrypted = reader + .decrypt_manifest_list_key_metadata(&encrypted_key_id) + .await + .unwrap(); + + assert_eq!(decrypted, key_metadata); + assert_eq!(generate_rule.num_calls(), 1); + assert_eq!(decrypt_rule.num_calls(), 1); } - fn test_client(client: Client) -> AwsKeyManagementClient { + fn test_client(client: aws_sdk_kms::Client) -> AwsKeyManagementClient { AwsKeyManagementClient::new( client, EncryptionAlgorithmSpec::SymmetricDefault, diff --git a/crates/kms/src/aws/config.rs b/crates/kms/src/aws/config.rs index 925c1aa4f5..cbcaeb782e 100644 --- a/crates/kms/src/aws/config.rs +++ b/crates/kms/src/aws/config.rs @@ -15,21 +15,16 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; + +use aws_sdk_kms::config::Credentials; use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; +use http::Uri; +use http::uri::Scheme; +use iceberg::io::{CLIENT_REGION, S3_ASSUME_ROLE_ARN}; use iceberg::{Error, ErrorKind, Result}; use iceberg_property_macro::Properties; -/// Catalog property selecting an AWS profile. -pub const AWS_PROFILE_NAME: &str = "profile_name"; -/// Catalog property selecting the AWS region. -pub const AWS_REGION_NAME: &str = "region_name"; -/// Catalog property containing a static AWS access key ID. -pub const AWS_ACCESS_KEY_ID: &str = "aws_access_key_id"; -/// Catalog property containing a static AWS secret access key. -pub const AWS_SECRET_ACCESS_KEY: &str = "aws_secret_access_key"; -/// Catalog property containing a static AWS session token. -pub const AWS_SESSION_TOKEN: &str = "aws_session_token"; - /// Catalog property overriding the AWS KMS service endpoint. pub const KMS_ENDPOINT: &str = "kms.endpoint"; /// Catalog property selecting the AWS KMS encryption algorithm. @@ -41,9 +36,16 @@ pub const KMS_DATA_KEY_SPEC: &str = "kms.data-key-spec"; /// Default AWS KMS generated key size, matching Iceberg Java. pub const KMS_DATA_KEY_SPEC_DEFAULT: &str = "AES_256"; -#[derive(Clone, Properties)] +// Properties shared with the Glue and S3 Tables catalogs. +pub(crate) const AWS_PROFILE_NAME: &str = "profile_name"; +pub(crate) const AWS_REGION_NAME: &str = "region_name"; +pub(crate) const AWS_ACCESS_KEY_ID: &str = "aws_access_key_id"; +pub(crate) const AWS_SECRET_ACCESS_KEY: &str = "aws_secret_access_key"; +pub(crate) const AWS_SESSION_TOKEN: &str = "aws_session_token"; + +#[derive(Properties)] pub(crate) struct AwsKmsConfig { - #[property(key = KMS_ENDPOINT, default = None)] + #[property(key = KMS_ENDPOINT, default = None, parse_with = parse_endpoint)] pub(crate) endpoint: Option, #[property( key = KMS_ENCRYPTION_ALGORITHM_SPEC, @@ -59,6 +61,82 @@ pub(crate) struct AwsKmsConfig { pub(crate) data_key_spec: DataKeySpec, } +/// Catalog properties used to build the AWS SDK configuration when the +/// application does not supply one. +#[derive(Properties)] +pub(crate) struct AwsSdkProperties { + #[property( + key = CLIENT_REGION, + additional_keys = [AWS_REGION_NAME], + default = None, + parse_properties_with = parse_first_present + )] + pub(crate) region: Option, + #[property(key = AWS_PROFILE_NAME, default = None)] + pub(crate) profile_name: Option, + #[property(key = AWS_ACCESS_KEY_ID, default = None)] + access_key_id: Option, + #[property(key = AWS_SECRET_ACCESS_KEY, default = None)] + secret_access_key: Option, + #[property(key = AWS_SESSION_TOKEN, default = None)] + session_token: Option, + /// Not supported yet; rejected rather than ignored, so that the KMS client + /// never silently uses a different identity than the catalog's file IO. + #[property(key = S3_ASSUME_ROLE_ARN, default = None)] + pub(crate) assume_role_arn: Option, +} + +impl AwsSdkProperties { + /// Static credentials from the catalog properties, if any are set. + pub(crate) fn static_credentials(&self) -> Result> { + match ( + &self.access_key_id, + &self.secret_access_key, + &self.session_token, + ) { + (Some(access_key_id), Some(secret_access_key), session_token) => { + Ok(Some(Credentials::new( + access_key_id, + secret_access_key, + session_token.clone(), + None, + "catalog-properties", + ))) + } + (None, None, None) => Ok(None), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "AWS static credentials require both '{AWS_ACCESS_KEY_ID}' and '{AWS_SECRET_ACCESS_KEY}'" + ), + )), + } + } +} + +/// Accepts the endpoints the AWS SDK can apply to a request: `http` or `https` +/// URIs with a host. +fn parse_endpoint(value: &str) -> Result { + let invalid = || { + Error::new( + ErrorKind::DataInvalid, + format!("AWS KMS endpoint must be an http or https URL with a host: '{value}'"), + ) + }; + let uri = value + .parse::() + .map_err(|source| invalid().with_source(source))?; + let has_supported_scheme = uri + .scheme() + .is_some_and(|scheme| *scheme == Scheme::HTTP || *scheme == Scheme::HTTPS); + let has_host = uri.host().is_some_and(|host| !host.is_empty()); + if has_supported_scheme && has_host { + Ok(value.to_string()) + } else { + Err(invalid()) + } +} + fn parse_encryption_algorithm(value: &str) -> Result { EncryptionAlgorithmSpec::try_parse(value).map_err(|source| { Error::new( @@ -79,10 +157,22 @@ fn parse_data_key_spec(value: &str) -> Result { }) } +/// Uses the value of the first key present, so Iceberg Java's key takes +/// precedence over the Glue-compatible alias. +fn parse_first_present( + properties: &HashMap, + key: &str, + additional_keys: &[&str], + default: Option, +) -> Result> { + Ok(std::iter::once(key) + .chain(additional_keys.iter().copied()) + .find_map(|key| properties.get(key).cloned()) + .or(default)) +} + #[cfg(test)] mod tests { - use std::collections::HashMap; - use super::*; #[test] @@ -120,6 +210,59 @@ mod tests { assert_eq!(config.data_key_spec, DataKeySpec::Aes128); } + #[test] + fn test_reject_incomplete_static_credentials() { + for keys in [ + vec![AWS_ACCESS_KEY_ID], + vec![AWS_SECRET_ACCESS_KEY], + vec![AWS_SESSION_TOKEN], + vec![AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN], + ] { + let properties = keys + .iter() + .map(|key| (key.to_string(), "value".to_string())) + .collect(); + let sdk_properties = AwsSdkProperties::from_properties(&properties).unwrap(); + let Err(error) = sdk_properties.static_credentials() else { + panic!("incomplete credentials {keys:?} must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + } + + #[test] + fn test_endpoint_validation() { + for endpoint in [ + "http://localhost:4566", + "http://[::1]:4566", + "https://kms.eu-west-2.amazonaws.com", + "https://kms-fips.us-east-1.amazonaws.com/", + "HTTPS://kms.eu-west-2.amazonaws.com", + "http://localhost:4566/kms", + ] { + assert_eq!(parse_endpoint(endpoint).unwrap(), endpoint); + } + // Unparsable URIs make the SDK describe the request, body included, in + // its error. + for endpoint in [ + "", + "localhost:4566", + "http://", + "https:///path", + "http://a b", + "http://:4566", + "http://%zz", + "http://\u{fc}.example", + "ftp://host", + ] { + assert_eq!( + parse_endpoint(endpoint).unwrap_err().kind(), + ErrorKind::DataInvalid, + "{endpoint}" + ); + } + } + #[test] fn test_reject_unknown_values() { let bad_algorithm = HashMap::from([( diff --git a/crates/kms/src/aws/factory.rs b/crates/kms/src/aws/factory.rs index 9bbd338d18..4093758845 100644 --- a/crates/kms/src/aws/factory.rs +++ b/crates/kms/src/aws/factory.rs @@ -20,23 +20,19 @@ use std::fmt; use std::sync::Arc; use async_trait::async_trait; -use aws_config::{BehaviorVersion, SdkConfig}; -use aws_sdk_kms::Client as KmsClient; -use aws_sdk_kms::config::{Builder as KmsConfigBuilder, Credentials, Region}; +use aws_config::{BehaviorVersion, ConfigLoader, Region, SdkConfig}; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::io::S3_ASSUME_ROLE_ARN; use iceberg::{Error, ErrorKind, Result}; use super::client::AwsKeyManagementClient; -use super::config::{ - AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, - AwsKmsConfig, -}; +use super::config::{AwsKmsConfig, AwsSdkProperties}; /// Factory for AWS KMS-backed Iceberg key management clients. /// /// By default, AWS credentials and region are resolved through the AWS SDK's -/// standard provider chains. Catalog properties can override the profile, -/// region, or static credentials. Applications with custom credential or HTTP +/// standard provider chains. Catalog properties can override the region, +/// profile, or static credentials. Applications with custom credential or HTTP /// providers can supply a complete [`SdkConfig`] through /// [`with_sdk_config`](Self::with_sdk_config). #[derive(Clone, Default)] @@ -52,9 +48,9 @@ impl AwsKmsClientFactory { /// Use an application-provided AWS SDK configuration. /// - /// When set, SDK-wide credential, region, retry, and HTTP configuration - /// properties are not rebuilt from catalog properties. KMS-specific - /// properties such as `kms.endpoint` continue to be applied. + /// When set, the region, profile, and credential catalog properties are not + /// read. KMS-specific properties such as `kms.endpoint` continue to be + /// applied. pub fn with_sdk_config(mut self, sdk_config: SdkConfig) -> Self { self.sdk_config = Some(sdk_config); self @@ -65,7 +61,7 @@ impl fmt::Debug for AwsKmsClientFactory { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("AwsKmsClientFactory") .field("has_sdk_config", &self.sdk_config.is_some()) - .finish() + .finish_non_exhaustive() } } @@ -78,10 +74,17 @@ impl KmsClientFactory for AwsKmsClientFactory { let kms_config = AwsKmsConfig::from_properties(properties)?; let sdk_config = match &self.sdk_config { Some(config) => config.clone(), - None => create_sdk_config(properties).await?, + None => { + let sdk_properties = AwsSdkProperties::from_properties(properties)?; + create_sdk_config( + &sdk_properties, + aws_config::defaults(BehaviorVersion::latest()), + ) + .await? + } }; - let mut builder = KmsConfigBuilder::from(&sdk_config); + let mut builder = aws_sdk_kms::config::Builder::from(&sdk_config); if sdk_config.behavior_version().is_none() { builder = builder.behavior_version(BehaviorVersion::latest()); } @@ -90,110 +93,163 @@ impl KmsClientFactory for AwsKmsClientFactory { } Ok(Arc::new(AwsKeyManagementClient::new( - KmsClient::from_conf(builder.build()), + aws_sdk_kms::Client::from_conf(builder.build()), kms_config.encryption_algorithm, kms_config.data_key_spec, ))) } } -async fn create_sdk_config(properties: &HashMap) -> Result { - let mut loader = aws_config::defaults(BehaviorVersion::latest()); - - match ( - properties.get(AWS_ACCESS_KEY_ID), - properties.get(AWS_SECRET_ACCESS_KEY), - ) { - (Some(access_key), Some(secret_key)) => { - let credentials = Credentials::new( - access_key, - secret_key, - properties.get(AWS_SESSION_TOKEN).cloned(), - None, - "catalog-properties", - ); - loader = loader.credentials_provider(credentials); - } - (None, None) => { - if properties.contains_key(AWS_SESSION_TOKEN) { - return Err(incomplete_credentials_error()); - } - } - _ => return Err(incomplete_credentials_error()), +async fn create_sdk_config( + properties: &AwsSdkProperties, + mut loader: ConfigLoader, +) -> Result { + if properties.assume_role_arn.is_some() { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "'{S3_ASSUME_ROLE_ARN}' is not supported by the AWS KMS client yet; supply an SdkConfig with the desired credentials through AwsKmsClientFactory::with_sdk_config" + ), + )); } - - if let Some(profile) = properties.get(AWS_PROFILE_NAME) { + if let Some(credentials) = properties.static_credentials()? { + loader = loader.credentials_provider(credentials); + } + if let Some(profile) = &properties.profile_name { loader = loader.profile_name(profile); } - if let Some(region) = properties.get(AWS_REGION_NAME) { + if let Some(region) = &properties.region { loader = loader.region(Region::new(region.clone())); } - Ok(loader.load().await) } -fn incomplete_credentials_error() -> Error { - Error::new( - ErrorKind::DataInvalid, - format!( - "AWS static credentials require both '{AWS_ACCESS_KEY_ID}' and '{AWS_SECRET_ACCESS_KEY}'" - ), - ) -} - #[cfg(test)] mod tests { - use aws_config::SdkConfig; - use aws_sdk_kms::config::ProvideCredentials; - use iceberg::encryption::kms::KmsClientFactory; + use aws_sdk_kms::config::{Credentials, ProvideCredentials}; + use aws_smithy_http_client::test_util::{CaptureRequestReceiver, capture_request}; + use base64::Engine as _; + use base64::prelude::BASE64_STANDARD; + use iceberg::io::CLIENT_REGION; use super::*; - use crate::aws::KMS_ENDPOINT; + use crate::aws::config::{ + AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + }; + use crate::aws::{KMS_DATA_KEY_SPEC, KMS_ENCRYPTION_ALGORITHM_SPEC, KMS_ENDPOINT}; + + const ROLE_ARN: &str = "arn:aws:iam::123456789012:role/kms"; + + /// A config loader that ignores the host's `AWS_*` environment variables + /// and shared configuration files. + fn test_loader() -> ConfigLoader { + aws_config::defaults(BehaviorVersion::latest()).empty_test_environment() + } + + /// A fully loaded SDK configuration whose HTTP client captures the first + /// request instead of sending it. + async fn capturing_sdk_config() -> (SdkConfig, CaptureRequestReceiver) { + let (http_client, request) = capture_request(None); + let sdk_config = test_loader() + .region(Region::new("eu-west-2")) + .credentials_provider(Credentials::new("access", "secret", None, None, "test")) + .http_client(http_client) + .load() + .await; + (sdk_config, request) + } + + async fn capturing_factory() -> (AwsKmsClientFactory, CaptureRequestReceiver) { + let (sdk_config, request) = capturing_sdk_config().await; + ( + AwsKmsClientFactory::new().with_sdk_config(sdk_config), + request, + ) + } + + fn request_body(request: CaptureRequestReceiver) -> String { + let request = request.expect_request(); + String::from_utf8(request.body().bytes().unwrap().to_vec()).unwrap() + } + + fn sdk_properties(properties: &[(&str, &str)]) -> AwsSdkProperties { + let properties = properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + AwsSdkProperties::from_properties(&properties).unwrap() + } #[tokio::test] - async fn test_create_client_with_injected_sdk_config() { - let factory = AwsKmsClientFactory::new().with_sdk_config(test_sdk_config()); + async fn test_kms_endpoint_overrides_injected_sdk_config() { + let (factory, request) = capturing_factory().await; let properties = HashMap::from([( KMS_ENDPOINT.to_string(), "http://localhost:4566".to_string(), )]); let client = factory.create_kms_client(&properties).await.unwrap(); - assert!(client.supports_key_generation()); + // The captured response is empty, so only the outgoing request matters here. + let _ = client.unwrap_key(b"wrapped", "test-key").await; + + assert_eq!(request.expect_request().uri(), "http://localhost:4566/"); } #[tokio::test] - async fn test_create_client_adds_missing_behavior_version() { - let factory = AwsKmsClientFactory::new().with_sdk_config(SdkConfig::builder().build()); + async fn test_reject_invalid_kms_endpoint() { + let factory = AwsKmsClientFactory::new().with_sdk_config(capturing_sdk_config().await.0); + let properties = HashMap::from([(KMS_ENDPOINT.to_string(), "localhost:4566".to_string())]); - let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + let Err(error) = factory.create_kms_client(&properties).await else { + panic!("endpoints without a scheme must be rejected"); + }; - assert!(client.supports_key_generation()); + assert_eq!(error.kind(), ErrorKind::DataInvalid); } #[tokio::test] - async fn test_reject_incomplete_static_credentials() { - let only_access_key = - HashMap::from([(AWS_ACCESS_KEY_ID.to_string(), "access".to_string())]); - let error = create_sdk_config(&only_access_key).await.unwrap_err(); - assert_eq!(error.kind(), ErrorKind::DataInvalid); + async fn test_factory_applies_kms_data_key_spec() { + let (factory, request) = capturing_factory().await; + let properties = HashMap::from([(KMS_DATA_KEY_SPEC.to_string(), "AES_128".to_string())]); - let only_session_token = - HashMap::from([(AWS_SESSION_TOKEN.to_string(), "token".to_string())]); - let error = create_sdk_config(&only_session_token).await.unwrap_err(); - assert_eq!(error.kind(), ErrorKind::DataInvalid); + let client = factory.create_kms_client(&properties).await.unwrap(); + let _ = client.generate_key("test-key").await; + + assert!(request_body(request).contains(r#""KeySpec":"AES_128""#)); + } + + #[tokio::test] + async fn test_factory_applies_kms_encryption_algorithm_spec() { + let (factory, request) = capturing_factory().await; + let properties = HashMap::from([( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "RSAES_OAEP_SHA_256".to_string(), + )]); + + let client = factory.create_kms_client(&properties).await.unwrap(); + let _ = client.wrap_key(b"key", "test-key").await; + + assert!(request_body(request).contains(r#""EncryptionAlgorithm":"RSAES_OAEP_SHA_256""#)); + } + + #[tokio::test] + async fn test_create_client_adds_missing_behavior_version() { + let factory = AwsKmsClientFactory::new().with_sdk_config(SdkConfig::builder().build()); + + // `aws_sdk_kms::Client::from_conf` panics if no behavior version is set. + factory.create_kms_client(&HashMap::new()).await.unwrap(); } #[tokio::test] async fn test_static_credentials_and_region_override_default_chains() { - let properties = HashMap::from([ - (AWS_ACCESS_KEY_ID.to_string(), "access".to_string()), - (AWS_SECRET_ACCESS_KEY.to_string(), "secret".to_string()), - (AWS_SESSION_TOKEN.to_string(), "token".to_string()), - (AWS_REGION_NAME.to_string(), "eu-west-2".to_string()), + let properties = sdk_properties(&[ + (AWS_ACCESS_KEY_ID, "access"), + (AWS_SECRET_ACCESS_KEY, "secret"), + (AWS_SESSION_TOKEN, "token"), + (AWS_REGION_NAME, "eu-west-2"), ]); - let config = create_sdk_config(&properties).await.unwrap(); + let config = create_sdk_config(&properties, test_loader()).await.unwrap(); let credentials = config .credentials_provider() .expect("static credentials must install a credentials provider") @@ -207,9 +263,66 @@ mod tests { assert_eq!(credentials.session_token(), Some("token")); } - fn test_sdk_config() -> SdkConfig { - SdkConfig::builder() - .behavior_version(BehaviorVersion::latest()) - .build() + #[tokio::test] + async fn test_client_region_takes_precedence_over_region_name() { + let properties = + sdk_properties(&[(CLIENT_REGION, "eu-west-1"), (AWS_REGION_NAME, "eu-west-2")]); + + let config = create_sdk_config(&properties, test_loader()).await.unwrap(); + + assert_eq!(config.region().map(Region::as_ref), Some("eu-west-1")); + } + + #[tokio::test] + async fn test_reject_assume_role() { + let properties = sdk_properties(&[(S3_ASSUME_ROLE_ARN, ROLE_ARN)]); + + let error = create_sdk_config(&properties, test_loader()) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn test_injected_sdk_config_ignores_sdk_properties() { + let factory = AwsKmsClientFactory::new().with_sdk_config(capturing_sdk_config().await.0); + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "access".to_string()), + (S3_ASSUME_ROLE_ARN.to_string(), ROLE_ARN.to_string()), + ]); + + assert!(factory.create_kms_client(&properties).await.is_ok()); + } + + #[tokio::test] + async fn test_invalid_sdk_endpoint_does_not_leak_plaintext_key() { + // The SDK describes the whole request, body included, when it cannot + // apply an endpoint URL. An injected `SdkConfig` is not validated. + let (http_client, _request) = capture_request(None); + let sdk_config = test_loader() + .region(Region::new("eu-west-2")) + .credentials_provider(Credentials::new("access", "secret", None, None, "test")) + .endpoint_url("http://%zz") + .http_client(http_client) + .load() + .await; + let factory = AwsKmsClientFactory::new().with_sdk_config(sdk_config); + let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + let plaintext = b"0123456789abcdef"; + let encoded = BASE64_STANDARD.encode(plaintext); + + let error = client.wrap_key(plaintext, "test-key").await.unwrap_err(); + + // Guards against this passing without the SDK describing the request. + assert!(error.message().contains(""), "{error}"); + + for output in [ + error.to_string(), + format!("{error:?}"), + format!("{error:#?}"), + ] { + assert!(!output.contains(&encoded), "{output}"); + } } } diff --git a/crates/kms/src/aws/mod.rs b/crates/kms/src/aws/mod.rs index 9782e7ac2b..c1f10954cd 100644 --- a/crates/kms/src/aws/mod.rs +++ b/crates/kms/src/aws/mod.rs @@ -17,39 +17,71 @@ //! AWS Key Management Service integration. //! -//! This module provides [`AwsKmsClientFactory`](crate::aws::AwsKmsClientFactory), +//! This module provides [`AwsKmsClientFactory`], //! which creates AWS-backed implementations of Iceberg's //! [`KeyManagementClient`](iceberg::encryption::KeyManagementClient). The factory can //! be supplied to any Iceberg catalog builder through //! [`CatalogBuilder::with_kms_client_factory`](iceberg::CatalogBuilder::with_kms_client_factory). //! +//! # Permissions +//! //! The KMS key referenced by a table's `encryption.key-id` property requires //! `kms:Decrypt` for reads. Writes with `SYMMETRIC_DEFAULT` use //! `kms:GenerateDataKey`. Writes with an asymmetric encryption algorithm //! generate the key-encryption key locally and require `kms:Encrypt` to wrap it. //! -//! # Configuration +//! Iceberg Java always generates key-encryption keys locally and wraps them +//! with `kms:Encrypt`, so key policies written for Java writers must also grant +//! `kms:GenerateDataKey` to Rust writers using `SYMMETRIC_DEFAULT`. Tables +//! written by either implementation are readable by the other. //! -//! The factory uses the AWS SDK's default credential and region provider chains. -//! The `profile_name`, `region_name`, `aws_access_key_id`, -//! `aws_secret_access_key`, and `aws_session_token` catalog properties can -//! override those defaults, following the conventions used by Iceberg Rust's -//! other AWS integrations. +//! # Configuration //! -//! Catalog properties are the properties supplied by the application when the -//! catalog is constructed. Configuration returned later by a REST catalog +//! The factory reads the catalog properties supplied by the application when +//! the catalog is constructed. Configuration returned later by a REST catalog //! server is not currently available when the KMS client is created. //! -//! The following KMS properties match Iceberg Java: +//! These KMS properties match Iceberg Java: //! -//! - `kms.endpoint` overrides the KMS service endpoint. +//! - `kms.endpoint` overrides the KMS service endpoint. It must be an `http` or +//! `https` URL. //! - `kms.encryption-algorithm-spec` selects the KMS encryption algorithm and //! defaults to `SYMMETRIC_DEFAULT`. -//! - `kms.data-key-spec` selects the size of KMS-generated key-encryption keys -//! and defaults to `AES_256`. +//! - `kms.data-key-spec` selects the size of key-encryption keys generated by +//! KMS and defaults to `AES_256`. It only applies to `SYMMETRIC_DEFAULT`; +//! with an asymmetric algorithm, locally generated key-encryption keys are +//! sized by the table property `encryption.data-key-length`, like data keys. +//! +//! Unless an [`SdkConfig`](aws_config::SdkConfig) is supplied through +//! [`AwsKmsClientFactory::with_sdk_config`], in which case they are not read, +//! the AWS SDK's default credential and region provider chains are used, and +//! these properties override them: +//! +//! - `client.region` selects the region, as in Iceberg Java. The `region_name` +//! property used by the Glue and S3 Tables catalogs is also accepted, with +//! lower precedence. +//! - `profile_name` selects a profile from the shared AWS configuration files. +//! - `aws_access_key_id`, `aws_secret_access_key`, and the optional +//! `aws_session_token` provide static credentials. +//! +//! Assuming an IAM role through `client.assume-role.*` is not supported yet: +//! catalog properties that set `client.assume-role.arn` are rejected, rather +//! than having the KMS client use a different identity than the catalog's S3 +//! file IO. Supply an [`SdkConfig`](aws_config::SdkConfig) with the desired +//! credentials instead. +//! +//! `client.credentials-provider` and `client.factory` name Java classes and +//! are ignored, as are the `s3.*` credentials used by S3 file IO. +//! +//! # Security considerations +//! +//! With an asymmetric encryption algorithm, wrapping a key-encryption key only +//! requires the KMS public key. Anyone who can write table metadata can +//! therefore add a key-encryption key they know, without any KMS permissions; +//! `SYMMETRIC_DEFAULT` requires `kms:GenerateDataKey` or `kms:Encrypt` instead. //! -//! `kms.data-key-spec` is independent from the Iceberg table property -//! `encryption.data-key-length`, which controls locally generated file data keys. +//! With an asymmetric encryption algorithm, plaintext key-encryption keys are +//! sent to KMS, and the AWS SDK's `TRACE` logs include request bodies. //! //! # Example //! @@ -58,13 +90,23 @@ //! use std::sync::Arc; //! //! use iceberg::CatalogBuilder; -//! use iceberg_catalog_rest::RestCatalogBuilder; +//! use iceberg::io::CLIENT_REGION; +//! use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; //! use iceberg_kms::aws::AwsKmsClientFactory; //! //! # async fn example() -> iceberg::Result<()> { -//! let catalog = RestCatalogBuilder::default() +//! let properties = HashMap::from([ +//! ( +//! MEMORY_CATALOG_WAREHOUSE.to_string(), +//! "s3://bucket/warehouse".to_string(), +//! ), +//! (CLIENT_REGION.to_string(), "eu-west-2".to_string()), +//! ]); +//! // Any catalog builder accepts a KMS client factory. A storage factory for +//! // the warehouse is omitted for brevity. +//! let catalog = MemoryCatalogBuilder::default() //! .with_kms_client_factory(Arc::new(AwsKmsClientFactory::new())) -//! .load("rest", HashMap::new()) +//! .load("catalog", properties) //! .await?; //! # let _ = catalog; //! # Ok(()) @@ -76,7 +118,6 @@ mod config; mod factory; pub use config::{ - AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, KMS_DATA_KEY_SPEC, KMS_DATA_KEY_SPEC_DEFAULT, KMS_ENCRYPTION_ALGORITHM_SPEC, KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT, KMS_ENDPOINT, }; diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs index 33b06c53c4..7269472f8a 100644 --- a/crates/kms/src/lib.rs +++ b/crates/kms/src/lib.rs @@ -22,6 +22,5 @@ #![deny(missing_docs)] -/// AWS Key Management Service integration. #[cfg(feature = "aws")] pub mod aws; From a09c378e7845ce31e14462552a337ea92c029396 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Wed, 7 Oct 2026 15:50:31 +0100 Subject: [PATCH 7/8] chore(kms): regenerate dependency list after merging main --- crates/kms/DEPENDENCIES.rust.tsv | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/kms/DEPENDENCIES.rust.tsv b/crates/kms/DEPENDENCIES.rust.tsv index da201d49dd..567a13ff0a 100644 --- a/crates/kms/DEPENDENCIES.rust.tsv +++ b/crates/kms/DEPENDENCIES.rust.tsv @@ -71,13 +71,11 @@ derive_builder_core@0.20.2 X X derive_builder_macro@0.20.2 X X digest@0.10.7 X X displaydoc@0.2.6 X X -dissimilar@1.0.11 X either@1.16.0 X X equivalent@1.0.2 X X erased-serde@0.4.10 X X event-listener@5.4.1 X X event-listener-strategy@0.5.4 X X -expect-test@1.5.1 X X fastnum@0.7.5 X X fastrand@2.4.1 X X find-msvc-tools@0.1.9 X X From 0d7a437fc1b637e39d69c6bbb98c529ba7cc3203 Mon Sep 17 00:00:00 2001 From: Zakariya Stasa Date: Wed, 7 Oct 2026 17:02:32 +0100 Subject: [PATCH 8/8] test(kms): exercise factory rejections and tidy configuration docs Test the assume-role and incomplete static-credential rejections through the factory, rename a test that no longer overrides anything, and clarify how catalog properties interact with an injected SdkConfig. --- crates/kms/src/aws/factory.rs | 27 +++++++++++++++++++++++---- crates/kms/src/aws/mod.rs | 8 ++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/crates/kms/src/aws/factory.rs b/crates/kms/src/aws/factory.rs index 4093758845..4ea003c302 100644 --- a/crates/kms/src/aws/factory.rs +++ b/crates/kms/src/aws/factory.rs @@ -241,7 +241,7 @@ mod tests { } #[tokio::test] - async fn test_static_credentials_and_region_override_default_chains() { + async fn test_static_credentials_and_region_are_applied() { let properties = sdk_properties(&[ (AWS_ACCESS_KEY_ID, "access"), (AWS_SECRET_ACCESS_KEY, "secret"), @@ -275,15 +275,34 @@ mod tests { #[tokio::test] async fn test_reject_assume_role() { - let properties = sdk_properties(&[(S3_ASSUME_ROLE_ARN, ROLE_ARN)]); + // Rejected before the AWS environment is read. + let properties = HashMap::from([(S3_ASSUME_ROLE_ARN.to_string(), ROLE_ARN.to_string())]); - let error = create_sdk_config(&properties, test_loader()) + let Err(error) = AwsKmsClientFactory::new() + .create_kms_client(&properties) .await - .unwrap_err(); + else { + panic!("assume-role properties must be rejected"); + }; assert_eq!(error.kind(), ErrorKind::FeatureUnsupported); } + #[tokio::test] + async fn test_reject_incomplete_static_credentials() { + // Rejected before the AWS environment is read. + let properties = HashMap::from([(AWS_ACCESS_KEY_ID.to_string(), "access".to_string())]); + + let Err(error) = AwsKmsClientFactory::new() + .create_kms_client(&properties) + .await + else { + panic!("incomplete static credentials must be rejected"); + }; + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + #[tokio::test] async fn test_injected_sdk_config_ignores_sdk_properties() { let factory = AwsKmsClientFactory::new().with_sdk_config(capturing_sdk_config().await.0); diff --git a/crates/kms/src/aws/mod.rs b/crates/kms/src/aws/mod.rs index c1f10954cd..3edf18f319 100644 --- a/crates/kms/src/aws/mod.rs +++ b/crates/kms/src/aws/mod.rs @@ -52,10 +52,10 @@ //! with an asymmetric algorithm, locally generated key-encryption keys are //! sized by the table property `encryption.data-key-length`, like data keys. //! -//! Unless an [`SdkConfig`](aws_config::SdkConfig) is supplied through -//! [`AwsKmsClientFactory::with_sdk_config`], in which case they are not read, -//! the AWS SDK's default credential and region provider chains are used, and -//! these properties override them: +//! The AWS SDK's default credential and region provider chains are used, and +//! the properties below override them. They are not read when an +//! [`SdkConfig`](aws_config::SdkConfig) is supplied through +//! [`AwsKmsClientFactory::with_sdk_config`]. //! //! - `client.region` selects the region, as in Iceberg Java. The `region_name` //! property used by the Glue and S3 Tables catalogs is also accepted, with