diff --git a/Cargo.lock b/Cargo.lock index e2900acb2e..6f9272fba6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -534,9 +534,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.18" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -544,11 +544,11 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-http 0.64.1", + "aws-smithy-json 0.63.1", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema 0.1.0", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", @@ -565,9 +565,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -600,14 +600,14 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" +checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", - "aws-smithy-http 0.63.6", + "aws-smithy-http 0.64.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -616,7 +616,7 @@ dependencies = [ "bytes-utils", "fastrand", "http 1.5.0", - "http-body 1.0.1", + "http-body", "percent-encoding", "pin-project-lite", "tracing", @@ -634,7 +634,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-json 0.62.7", "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -648,6 +648,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-kms" +version = "1.123.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bbe8833a7ad3b970f2a6a856939c9cb171942dcdfd2ab65f2db3a8baaf73cc1" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.64.1", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.5.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-s3tables" version = "1.59.0" @@ -659,7 +684,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-json 0.62.7", "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -675,24 +700,24 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.102.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" +checksum = "12be2f9c8eef7f5fc919c96d538e629698469a02b4cb75408b26b1bd984ebe79" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.1", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -700,24 +725,24 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.104.0" +version = "1.116.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" +checksum = "d7645db8724ea3b82fdccfb67e1b0f637c9d8ab0e7ef29d84884d8d8d73f805d" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.1", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -725,25 +750,25 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.107.0" +version = "1.119.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" +checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.1", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", - "http 0.2.12", "http 1.5.0", "regex-lite", "tracing", @@ -751,19 +776,18 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.5" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" +checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" dependencies = [ "aws-credential-types", - "aws-smithy-http 0.63.6", + "aws-smithy-http 0.64.1", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "form_urlencoded", "hex", "hmac 0.13.0", - "http 0.2.12", "http 1.5.0", "percent-encoding", "sha2 0.11.0", @@ -795,7 +819,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", "percent-encoding", "pin-project-lite", @@ -805,9 +829,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.0" +version = "0.64.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -816,7 +840,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", "percent-encoding", "pin-project-lite", @@ -826,22 +850,28 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.2.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" +checksum = "51c89cc3f1f281d659a67a519a1b5c6d445b5ce09fa7e5aee40c2c2707e9509d" dependencies = [ "aws-smithy-async", + "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", + "bytes", "h2", "http 1.5.0", + "http-body", "hyper", "hyper-rustls", "hyper-util", + "indexmap 2.14.0", "pin-project-lite", "rustls", "rustls-native-certs", "rustls-pki-types", + "serde", + "serde_json", "tokio", "tokio-rustls", "tower", @@ -859,6 +889,29 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-mocks" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0922f8ee1ce1f90f0baf9c9036f566161547a7ab9f09571d61d56455ba35598" +dependencies = [ + "aws-smithy-http-client", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + [[package]] name = "aws-smithy-observability" version = "0.2.6" @@ -877,35 +930,55 @@ dependencies = [ "aws-smithy-runtime-api", ] +[[package]] +name = "aws-smithy-protocol-test" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a" +dependencies = [ + "assert-json-diff", + "aws-smithy-runtime-api", + "base64-simd", + "cbor-diag", + "ciborium", + "http 0.2.12", + "pretty_assertions", + "regex-lite", + "roxmltree", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "aws-smithy-query" -version = "0.60.15" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.12.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" +checksum = "d6e302ac1d88b99652489df31abdec6ac42a2ab2ac3982ad0ac49f64dfaf28ba" dependencies = [ "aws-smithy-async", - "aws-smithy-http 0.64.0", + "aws-smithy-http 0.64.1", "aws-smithy-http-client", "aws-smithy-observability 0.3.0", "aws-smithy-runtime-api", - "aws-smithy-schema 0.2.0", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "bytes", "fastrand", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", - "http-body 1.0.1", + "http-body", "http-body-util", "pin-project-lite", "pin-utils", @@ -915,15 +988,14 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.13.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" +checksum = "c0730c16f91124c6a2abb4932c77e299288b3dd9f967ea2e9ec48cc6731e87a4" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", - "http 0.2.12", "http 1.5.0", "pin-project-lite", "tokio", @@ -955,9 +1027,9 @@ dependencies = [ [[package]] name = "aws-smithy-schema" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -966,18 +1038,16 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" +checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", - "http 0.2.12", "http 1.5.0", - "http-body 0.4.6", - "http-body 1.0.1", + "http-body", "http-body-util", "itoa", "num-integer", @@ -992,23 +1062,26 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.15" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.16" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema 0.1.0", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "rustc_version", "tracing", @@ -1265,6 +1338,25 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbor-diag" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" +dependencies = [ + "bs58", + "chrono", + "data-encoding", + "half", + "nom", + "num-bigint 0.4.8", + "num-rational", + "num-traits", + "separator", + "url", + "uuid", +] + [[package]] name = "cc" version = "1.2.66" @@ -1330,6 +1422,33 @@ dependencies = [ "phf", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -1776,6 +1895,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "defmt" version = "1.1.1" @@ -2546,17 +2671,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -2576,7 +2690,7 @@ dependencies = [ "bytes", "futures-core", "http 1.5.0", - "http-body 1.0.1", + "http-body", "pin-project-lite", ] @@ -2619,7 +2733,7 @@ dependencies = [ "futures-core", "h2", "http 1.5.0", - "http-body 1.0.1", + "http-body", "httparse", "httpdate", "itoa", @@ -2656,7 +2770,7 @@ dependencies = [ "futures-channel", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "hyper", "ipnet", "libc", @@ -2912,6 +3026,22 @@ dependencies = [ "uuid", ] +[[package]] +name = "iceberg-kms" +version = "0.10.1" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-kms", + "aws-smithy-http-client", + "aws-smithy-mocks", + "base64 0.22.1", + "http 1.5.0", + "iceberg", + "iceberg-property-macro", + "tokio", +] + [[package]] name = "iceberg-property-macro" version = "0.10.1" @@ -3564,6 +3694,12 @@ dependencies = [ "serde", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3622,7 +3758,7 @@ dependencies = [ "colored", "futures-core", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", "hyper", "hyper-util", @@ -3730,6 +3866,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" version = "0.4.3" @@ -3819,6 +3965,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3959,7 +4116,7 @@ dependencies = [ "bytes", "futures", "http 1.5.0", - "http-body 1.0.1", + "http-body", "opendal-core", "reqwest 0.13.4", ] @@ -5044,7 +5201,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", "hyper", "hyper-util", @@ -5079,7 +5236,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "http-body-util", "hyper", "hyper-rustls", @@ -5175,6 +5332,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + [[package]] name = "rsa" version = "0.9.10" @@ -5457,6 +5623,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "separator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" + [[package]] name = "seq-macro" version = "0.3.6" @@ -5533,6 +5705,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6193,7 +6366,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -6476,7 +6649,7 @@ dependencies = [ "bytes", "futures-util", "http 1.5.0", - "http-body 1.0.1", + "http-body", "pin-project-lite", "tower", "tower-layer", diff --git a/Cargo.toml b/Cargo.toml index 661c56b094..b3e0278151 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/iceberg", "crates/integration_tests", "crates/integrations/*", + "crates/kms", "crates/property-macro", "crates/storage/*", "crates/test_utils", @@ -59,15 +60,21 @@ arrow-select = "59.2" arrow-string = "59.2" as-any = "0.3.2" async-trait = "0.1.89" -aws-config = "1.8.7" +aws-config = "1.9" aws-sdk-glue = { version = "1.85", default-features = false, features = [ "default-https-client", "rt-tokio", ] } +aws-sdk-kms = { version = "1.123", default-features = false, features = [ + "default-https-client", + "rt-tokio", +] } aws-sdk-s3tables = { version = "1.28", default-features = false, features = [ "default-https-client", "rt-tokio", ] } +aws-smithy-http-client = "1.5" +aws-smithy-mocks = "0.3" backon = "1.5.1" base64 = "0.22.1" bimap = "0.6" diff --git a/README.md b/README.md index fa974dc767..ca2a96edad 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ The Apache Iceberg Rust project is composed of the following components: | [iceberg-catalog-s3tables] | [![iceberg-catalog-s3tables image]][iceberg-catalog-s3tables link] | [![docs release]][iceberg-catalog-s3tables release docs] [![docs dev]][iceberg-catalog-s3tables dev docs] | | [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | | [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | +| [iceberg-kms] | [![iceberg-kms image]][iceberg-kms link] | [![docs release]][iceberg-kms release docs] [![docs dev]][iceberg-kms dev docs] | | [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue @@ -95,6 +96,12 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg-cache-moka release docs]: https://docs.rs/iceberg-cache-moka [iceberg-cache-moka dev docs]: https://rust.iceberg.apache.org/api/iceberg_cache_moka/ +[iceberg-kms]: crates/kms/README.md +[iceberg-kms image]: https://img.shields.io/crates/v/iceberg-kms.svg +[iceberg-kms link]: https://crates.io/crates/iceberg-kms +[iceberg-kms release docs]: https://docs.rs/iceberg-kms +[iceberg-kms dev docs]: https://rust.iceberg.apache.org/api/iceberg_kms/ + ## Iceberg Rust Implementation Status The features that Iceberg Rust currently supports can be found [here](https://iceberg.apache.org/status/). diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml new file mode 100644 index 0000000000..6c91400480 --- /dev/null +++ b/crates/kms/Cargo.toml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-kms" +publish = true +readme = "README.md" +rust-version = { workspace = true } +version = { workspace = true } + +categories = ["database"] +description = "Key management service integrations for Apache Iceberg client-side encryption" +keywords = ["iceberg", "kms", "encryption"] +license = { workspace = true } +repository = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[features] +aws = [ + "dep:async-trait", + "dep:aws-config", + "dep:aws-sdk-kms", + "dep:base64", + "dep:http", + "dep:iceberg-property-macro", +] +default = [] + +[dependencies] +async-trait = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-sdk-kms = { workspace = true, optional = true } +base64 = { workspace = true, optional = true } +http = { workspace = true, optional = true } +iceberg = { workspace = true } +iceberg-property-macro = { workspace = true, optional = true } + +[dev-dependencies] +aws-sdk-kms = { workspace = true, features = ["test-util"] } +aws-smithy-http-client = { workspace = true, features = ["test-util"] } +aws-smithy-mocks = { workspace = true } +tokio = { workspace = true, features = ["macros"] } + +[lints] +workspace = true diff --git a/crates/kms/DEPENDENCIES.rust.tsv b/crates/kms/DEPENDENCIES.rust.tsv new file mode 100644 index 0000000000..567a13ff0a --- /dev/null +++ b/crates/kms/DEPENDENCIES.rust.tsv @@ -0,0 +1,298 @@ +crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib +adler2@2.0.1 X X X +aead@0.5.2 X X +aes@0.8.4 X X +aes-gcm@0.10.3 X X +ahash@0.8.12 X X +aho-corasick@1.1.4 X X +alloc-no-stdlib@2.0.4 X +alloc-stdlib@0.2.4 X +android_system_properties@0.1.5 X X +anyhow@1.0.103 X X +apache-avro@0.21.0 X +array-init@2.1.0 X X +arrow-arith@59.3.0 X +arrow-array@59.3.0 X X +arrow-buffer@59.3.0 X +arrow-cast@59.3.0 X +arrow-data@59.3.0 X +arrow-ipc@59.3.0 X +arrow-ord@59.3.0 X +arrow-schema@59.3.0 X +arrow-select@59.3.0 X +arrow-string@59.3.0 X +as-any@0.3.2 X X +async-lock@3.4.2 X X +async-trait@0.1.89 X X +atoi@2.0.0 X +atomic-waker@1.1.2 X X +autocfg@1.5.1 X X +backon@1.6.0 X +base64@0.22.1 X X +base64@0.23.0 X X +bigdecimal@0.4.10 X X +bimap@0.6.3 X X +bitflags@2.13.0 X X +block-buffer@0.10.4 X X +bnum@0.12.1 X X +bon@3.9.3 X X +bon-macros@3.9.3 X X +brotli@8.0.4 X X +brotli-decompressor@5.0.3 X X +bumpalo@3.20.3 X X +bytemuck@1.25.0 X X X +bytemuck_derive@1.10.2 X X X +byteorder@1.5.0 X X +bytes@1.12.0 X +cc@1.2.66 X X +cfg-if@1.0.4 X X +chrono@0.4.45 X X +cipher@0.4.4 X X +concurrent-queue@2.5.0 X X +const-random@0.1.18 X X +const-random-macro@0.1.16 X X +core-foundation-sys@0.8.7 X X +cpufeatures@0.2.17 X X +crc32fast@1.5.0 X X +crossbeam-channel@0.5.16 X X +crossbeam-epoch@0.9.20 X X +crossbeam-utils@0.8.22 X X +crunchy@0.2.4 X +crypto-common@0.1.7 X X +ctr@0.9.2 X X +darling@0.20.11 X +darling@0.23.0 X +darling_core@0.20.11 X +darling_core@0.23.0 X +darling_macro@0.20.11 X +darling_macro@0.23.0 X +derive_builder@0.20.2 X X +derive_builder_core@0.20.2 X X +derive_builder_macro@0.20.2 X X +digest@0.10.7 X X +displaydoc@0.2.6 X X +either@1.16.0 X X +equivalent@1.0.2 X X +erased-serde@0.4.10 X X +event-listener@5.4.1 X X +event-listener-strategy@0.5.4 X X +fastnum@0.7.5 X X +fastrand@2.4.1 X X +find-msvc-tools@0.1.9 X X +flatbuffers@25.12.19 X +flate2@1.1.9 X X +fnv@1.0.7 X X +form_urlencoded@1.2.2 X X +futures@0.3.32 X X +futures-channel@0.3.32 X X +futures-core@0.3.32 X X +futures-executor@0.3.32 X X +futures-io@0.3.32 X X +futures-macro@0.3.32 X X +futures-sink@0.3.32 X X +futures-task@0.3.32 X X +futures-util@0.3.32 X X +generic-array@0.14.7 X +getrandom@0.2.17 X X +getrandom@0.3.4 X X +getrandom@0.4.3 X X +ghash@0.5.1 X X +gloo-timers@0.3.0 X X +h2@0.4.16 X +half@2.7.1 X X +hashbrown@0.17.1 X X +heck@0.5.0 X X +http@1.5.0 X X +http-body@1.0.1 X +http-body-util@0.1.3 X +httparse@1.10.1 X X +hyper@1.10.1 X +hyper-util@0.1.20 X +iana-time-zone@0.1.65 X X +iana-time-zone-haiku@0.1.2 X X +iceberg@0.10.1 X +iceberg-kms@0.10.1 X +iceberg-property-macro@0.10.1 X +iceberg_test_utils@0.10.1 X +icu_collections@2.2.0 X +icu_locale_core@2.2.0 X +icu_normalizer@2.2.0 X +icu_normalizer_data@2.2.0 X +icu_properties@2.2.0 X +icu_properties_data@2.2.0 X +icu_provider@2.2.0 X +ident_case@1.0.1 X X +idna@1.1.0 X X +idna_adapter@1.2.2 X X +indexmap@2.14.0 X X +inout@0.1.4 X X +inventory@0.3.24 X X +ipnet@2.12.0 X X +itertools@0.13.0 X X +itoa@1.0.18 X X +jobserver@0.1.35 X X +js-sys@0.3.103 X X +lazy_static@1.5.0 X X +lexical-core@1.0.6 X X +lexical-parse-float@1.0.6 X X +lexical-parse-integer@1.0.6 X X +lexical-util@1.0.7 X X +lexical-write-float@1.0.6 X X +lexical-write-integer@1.0.6 X X +libc@0.2.186 X X +libm@0.2.16 X +litemap@0.8.2 X +lock_api@0.4.14 X X +log@0.4.33 X X +lz4_flex@0.14.0 X +memchr@2.8.2 X X +miniz_oxide@0.8.9 X X X +mio@1.2.1 X +moka@0.12.15 X X +murmur3@0.5.2 X X +nu-ansi-term@0.50.3 X +num-bigint@0.4.8 X X +num-bigint@0.5.1 X X +num-complex@0.4.6 X X +num-integer@0.1.46 X X +num-traits@0.2.19 X X +once_cell@1.21.4 X X +opaque-debug@0.3.1 X X +ordered-float@4.6.0 X +parking@2.2.1 X X +parking_lot@0.12.5 X X +parking_lot_core@0.9.12 X X +parquet@59.3.0 X +percent-encoding@2.3.2 X X +pin-project-lite@0.2.17 X X +pkg-config@0.3.33 X X +polyval@0.6.2 X X +portable-atomic@1.13.1 X X +potential_utf@0.1.5 X +ppv-lite86@0.2.21 X X +prettyplease@0.2.37 X X +proc-macro2@1.0.106 X X +quad-rand@0.2.3 X +quote@1.0.46 X X +r-efi@5.3.0 X X X +r-efi@6.0.0 X X X +rand@0.9.5 X X +rand_chacha@0.9.0 X X +rand_core@0.6.4 X X +rand_core@0.9.5 X X +redox_syscall@0.5.18 X +regex@1.13.1 X X +regex-automata@0.4.16 X X +regex-lite@0.1.9 X X +regex-syntax@0.8.11 X X +reqwest@0.12.28 X X +ring@0.17.14 X X +roaring@0.11.4 X X +rustc_version@0.4.1 X X +rustversion@1.0.23 X X +ryu@1.0.23 X X +scopeguard@1.2.0 X X +semver@1.0.28 X X +seq-macro@0.3.6 X X +serde@1.0.228 X X +serde-big-array@0.5.1 X X +serde_bytes@0.11.19 X X +serde_core@1.0.228 X X +serde_derive@1.0.228 X X +serde_json@1.0.150 X X +serde_repr@0.1.20 X X +serde_urlencoded@0.7.1 X X +serde_with@3.22.0 X X +serde_with_macros@3.22.0 X X +sharded-slab@0.1.7 X +shlex@2.0.1 X X +simd-adler32@0.3.9 X +simdutf8@0.1.5 X X +slab@0.4.12 X +smallvec@1.15.2 X X +snap@1.1.1 X +socket2@0.6.4 X X +stable_deref_trait@1.2.1 X X +strsim@0.11.1 X +strum@0.27.2 X +strum_macros@0.27.2 X +subtle@2.6.1 X +syn@2.0.119 X X +sync_wrapper@1.0.2 X +synstructure@0.13.2 X +tagptr@0.2.0 X X +thiserror@2.0.18 X X +thiserror-impl@2.0.18 X X +thread_local@1.1.9 X X +tiny-keccak@2.0.2 X +tinystr@0.8.3 X +tokio@1.53.1 X +tokio-macros@2.7.0 X +tokio-util@0.7.18 X +tower@0.5.3 X +tower-http@0.6.11 X +tower-layer@0.3.3 X +tower-service@0.3.3 X +tracing@0.1.44 X +tracing-attributes@0.1.31 X +tracing-core@0.1.36 X +tracing-log@0.2.0 X +tracing-subscriber@0.3.23 X +try-lock@0.2.5 X +twox-hash@2.1.2 X +typed-builder@0.20.1 X X +typed-builder-macro@0.20.1 X X +typeid@1.0.3 X X +typenum@1.20.1 X X +typetag@0.2.22 X X +typetag-impl@0.2.22 X X +unicode-ident@1.0.24 X X X +universal-hash@0.5.1 X X +untrusted@0.9.0 X +url@2.5.8 X X +utf8_iter@1.0.4 X X +uuid@1.26.0 X X +version_check@0.9.5 X X +want@0.3.1 X +wasi@0.11.1+wasi-snapshot-preview1 X X X +wasip2@1.0.4+wasi-0.2.12 X X X +wasm-bindgen@0.2.126 X X +wasm-bindgen-futures@0.4.76 X X +wasm-bindgen-macro@0.2.126 X X +wasm-bindgen-macro-support@0.2.126 X X +wasm-bindgen-shared@0.2.126 X X +web-sys@0.3.103 X X +windows-core@0.62.2 X X +windows-implement@0.60.2 X X +windows-interface@0.59.3 X X +windows-link@0.2.1 X X +windows-result@0.4.1 X X +windows-strings@0.5.1 X X +windows-sys@0.52.0 X X +windows-sys@0.61.2 X X +windows-targets@0.52.6 X X +windows_aarch64_gnullvm@0.52.6 X X +windows_aarch64_msvc@0.52.6 X X +windows_i686_gnu@0.52.6 X X +windows_i686_gnullvm@0.52.6 X X +windows_i686_msvc@0.52.6 X X +windows_x86_64_gnu@0.52.6 X X +windows_x86_64_gnullvm@0.52.6 X X +windows_x86_64_msvc@0.52.6 X X +wit-bindgen@0.57.1 X X X +writeable@0.6.3 X +yoke@0.8.3 X +yoke-derive@0.8.2 X +zerocopy@0.8.53 X X X +zerocopy-derive@0.8.53 X X X +zerofrom@0.1.8 X +zerofrom-derive@0.1.7 X +zeroize@1.9.0 X X +zerotrie@0.2.4 X +zerovec@0.11.6 X +zerovec-derive@0.11.3 X +zlib-rs@0.6.5 X +zmij@1.0.21 X +zstd@0.13.3 X +zstd-safe@7.2.4 X X +zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/kms/LICENSE b/crates/kms/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/crates/kms/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/kms/NOTICE b/crates/kms/NOTICE new file mode 120000 index 0000000000..fb376cfaa4 --- /dev/null +++ b/crates/kms/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/crates/kms/README.md b/crates/kms/README.md new file mode 100644 index 0000000000..44f77ddaf7 --- /dev/null +++ b/crates/kms/README.md @@ -0,0 +1,38 @@ + + +# Apache Iceberg KMS + +This crate provides key management service integrations for Apache Iceberg +client-side encryption. Provider implementations are enabled through feature +flags. + +| Provider | Feature flag | +| -------------------------- | ------------ | +| AWS Key Management Service | `aws` | + +Add the crate with the feature for the provider you need: + +```toml +[dependencies] +iceberg-kms = { version = "x.y.z", features = ["aws"] } +``` + +See the [API documentation](https://docs.rs/iceberg-kms/latest) for +provider-specific configuration and usage. diff --git a/crates/kms/public-api.txt b/crates/kms/public-api.txt new file mode 100644 index 0000000000..34c635f643 --- /dev/null +++ b/crates/kms/public-api.txt @@ -0,0 +1,19 @@ +pub mod iceberg_kms +pub mod iceberg_kms::aws +pub struct iceberg_kms::aws::AwsKmsClientFactory +impl iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::new() -> Self +pub fn iceberg_kms::aws::AwsKmsClientFactory::with_sdk_config(self, sdk_config: aws_types::sdk_config::SdkConfig) -> Self +impl core::clone::Clone for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::clone(&self) -> iceberg_kms::aws::AwsKmsClientFactory +impl core::default::Default for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::default() -> iceberg_kms::aws::AwsKmsClientFactory +impl core::fmt::Debug for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::kms::factory::KmsClientFactory for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC: &str +pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC_DEFAULT: &str +pub const iceberg_kms::aws::KMS_ENCRYPTION_ALGORITHM_SPEC: &str +pub const iceberg_kms::aws::KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str +pub const iceberg_kms::aws::KMS_ENDPOINT: &str diff --git a/crates/kms/src/aws/client.rs b/crates/kms/src/aws/client.rs new file mode 100644 index 0000000000..47ce97408e --- /dev/null +++ b/crates/kms/src/aws/client.rs @@ -0,0 +1,713 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt; + +use async_trait::async_trait; +use aws_sdk_kms::error::{ProvideErrorMetadata, SdkError}; +use aws_sdk_kms::primitives::Blob; +use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; +use base64::Engine as _; +use base64::prelude::BASE64_STANDARD; +use iceberg::encryption::SensitiveBytes; +use iceberg::encryption::kms::{GeneratedKey, KeyManagementClient}; +use iceberg::{Error, ErrorKind, Result}; + +pub(crate) struct AwsKeyManagementClient { + client: aws_sdk_kms::Client, + encryption_algorithm: EncryptionAlgorithmSpec, + data_key_spec: DataKeySpec, +} + +impl AwsKeyManagementClient { + pub(crate) fn new( + client: aws_sdk_kms::Client, + encryption_algorithm: EncryptionAlgorithmSpec, + data_key_spec: DataKeySpec, + ) -> Self { + Self { + client, + encryption_algorithm, + data_key_spec, + } + } + + /// Converts a failed KMS request into an Iceberg error. + /// + /// `plaintext` is the key sent in the request, if any. The SDK can describe + /// the request, including its body, in transport errors (for example when + /// an endpoint URL cannot be applied), so its base64 encoding is redacted + /// from the message. The raw HTTP response, which holds the plaintext key + /// when a Decrypt or GenerateDataKey response fails to parse, is dropped. + fn request_failed( + operation: &str, + wrapping_key_id: &str, + plaintext: Option<&[u8]>, + error: SdkError, + ) -> Error + where + E: ProvideErrorMetadata + std::error::Error + Send + Sync + 'static, + R: fmt::Debug, + { + let message = |cause: String| { + format!("AWS KMS {operation} failed for key '{wrapping_key_id}': {cause}") + }; + match error { + SdkError::ServiceError(context) => { + let service_error = context.into_err(); + let description = service_error.to_string(); + let cause = match service_error.message() { + Some(aws_message) if !description.contains(aws_message) => { + format!("{description}: {aws_message}") + } + _ => description, + }; + Error::new(ErrorKind::Unexpected, message(cause)).with_source(service_error) + } + error => { + let mut causes = + std::iter::successors(Some(&error as &dyn std::error::Error), |error| { + (*error).source() + }) + .map(ToString::to_string) + .collect::>(); + causes.dedup(); + let cause = causes.join(": "); + let cause = match plaintext.filter(|plaintext| !plaintext.is_empty()) { + Some(plaintext) => { + cause.replace(&BASE64_STANDARD.encode(plaintext), "") + } + None => cause, + }; + // The cause chain is not attached as a source, since only the + // message is redacted. + Error::new(ErrorKind::Unexpected, message(cause)) + } + } + } + + fn missing_response_field(operation: &str, field: &str) -> Error { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS {operation} response did not contain {field}"), + ) + } + + fn ciphertext_blob(operation: &str, blob: Option) -> Result> { + blob.map(Blob::into_inner) + .filter(|ciphertext| !ciphertext.is_empty()) + .ok_or_else(|| Self::missing_response_field(operation, "ciphertext_blob")) + } + + fn generated_key_length(&self) -> Option { + match self.data_key_spec { + DataKeySpec::Aes128 => Some(16), + DataKeySpec::Aes256 => Some(32), + _ => None, + } + } +} + +impl fmt::Debug for AwsKeyManagementClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AwsKeyManagementClient") + .field("encryption_algorithm", &self.encryption_algorithm) + .field("data_key_spec", &self.data_key_spec) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl KeyManagementClient for AwsKeyManagementClient { + async fn wrap_key(&self, key: &[u8], wrapping_key_id: &str) -> Result> { + let response = self + .client + .encrypt() + .key_id(wrapping_key_id) + .encryption_algorithm(self.encryption_algorithm.clone()) + .plaintext(Blob::new(key)) + .send() + .await + .map_err(|source| { + Self::request_failed("Encrypt", wrapping_key_id, Some(key), source) + })?; + + Self::ciphertext_blob("Encrypt", response.ciphertext_blob) + } + + async fn unwrap_key( + &self, + wrapped_key: &[u8], + wrapping_key_id: &str, + ) -> Result { + let response = self + .client + .decrypt() + .key_id(wrapping_key_id) + .encryption_algorithm(self.encryption_algorithm.clone()) + .ciphertext_blob(Blob::new(wrapped_key)) + .send() + .await + .map_err(|source| Self::request_failed("Decrypt", wrapping_key_id, None, source))?; + + response + .plaintext + .map(|blob| SensitiveBytes::new(blob.into_inner())) + .ok_or_else(|| Self::missing_response_field("Decrypt", "plaintext")) + } + + fn supports_key_generation(&self) -> bool { + self.encryption_algorithm == EncryptionAlgorithmSpec::SymmetricDefault + } + + async fn generate_key(&self, wrapping_key_id: &str) -> Result { + let response = self + .client + .generate_data_key() + .key_id(wrapping_key_id) + .key_spec(self.data_key_spec.clone()) + .send() + .await + .map_err(|source| { + Self::request_failed("GenerateDataKey", wrapping_key_id, None, source) + })?; + + let plaintext = response + .plaintext + .map(|blob| SensitiveBytes::new(blob.into_inner())) + .ok_or_else(|| Self::missing_response_field("GenerateDataKey", "plaintext"))?; + if let Some(expected) = self.generated_key_length() + && plaintext.len() != expected + { + return Err(Error::new( + ErrorKind::Unexpected, + format!( + "AWS KMS GenerateDataKey returned a {}-byte key for key spec {}, expected {expected} bytes", + plaintext.len(), + self.data_key_spec.as_str() + ), + )); + } + let wrapped_key = Self::ciphertext_blob("GenerateDataKey", response.ciphertext_blob)?; + + Ok(GeneratedKey::new(plaintext, wrapped_key)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use aws_sdk_kms::error::{ConnectorError, ErrorMetadata}; + use aws_sdk_kms::operation::decrypt::{DecryptError, DecryptOutput}; + use aws_sdk_kms::operation::encrypt::{EncryptError, EncryptOutput}; + use aws_sdk_kms::operation::generate_data_key::{GenerateDataKeyError, GenerateDataKeyOutput}; + use aws_sdk_kms::types::error::{DisabledException, NotFoundException}; + use aws_smithy_mocks::{RuleMode, mock, mock_client}; + use iceberg::encryption::{EncryptionManager, StandardKeyMetadata}; + + use super::*; + + const KEY_ID: &str = "arn:aws:kms:eu-west-2:123456789012:key/test-key"; + + #[tokio::test] + async fn test_wrap_key() { + let plaintext = b"0123456789abcdef"; + let ciphertext = b"wrapped-key"; + let rule = mock!(aws_sdk_kms::Client::encrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::SymmetricDefault) + && request.plaintext().map(AsRef::as_ref) == Some(plaintext.as_slice()) + }) + .then_output(move || { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(ciphertext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + assert_eq!(kms.wrap_key(plaintext, KEY_ID).await.unwrap(), ciphertext); + assert_eq!(rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_unwrap_key() { + let plaintext = b"0123456789abcdef"; + let ciphertext = b"wrapped-key"; + let rule = mock!(aws_sdk_kms::Client::decrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::SymmetricDefault) + && request.ciphertext_blob().map(AsRef::as_ref) == Some(ciphertext.as_slice()) + }) + .then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + assert_eq!( + kms.unwrap_key(ciphertext, KEY_ID).await.unwrap().as_bytes(), + plaintext + ); + assert_eq!(rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_generate_key() { + let plaintext = b"0123456789abcdef0123456789abcdef"; + let ciphertext = b"generated-wrapped-key"; + let rule = mock!(aws_sdk_kms::Client::generate_data_key) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) && request.key_spec() == Some(&DataKeySpec::Aes256) + }) + .then_output(move || { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new(plaintext)) + .ciphertext_blob(Blob::new(ciphertext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + let generated = kms.generate_key(KEY_ID).await.unwrap(); + assert_eq!(generated.key().as_bytes(), plaintext); + assert_eq!(generated.wrapped_key(), ciphertext); + assert_eq!(rule.num_calls(), 1); + } + + #[test] + fn test_key_generation_support_depends_on_encryption_algorithm() { + let client = mock_client!(aws_sdk_kms, &[]); + + assert!( + AwsKeyManagementClient::new( + client.clone(), + EncryptionAlgorithmSpec::SymmetricDefault, + DataKeySpec::Aes256, + ) + .supports_key_generation() + ); + assert!( + !AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::RsaesOaepSha256, + DataKeySpec::Aes256, + ) + .supports_key_generation() + ); + } + + #[tokio::test] + async fn test_asymmetric_manager_roundtrip_uses_encrypt_instead_of_generate_data_key() { + let plaintext_kek = Arc::new(Mutex::new(Vec::new())); + let plaintext_kek_from_encrypt = Arc::clone(&plaintext_kek); + let encrypt_rule = mock!(aws_sdk_kms::Client::encrypt) + .match_requests(move |request| { + if request.key_id() != Some(KEY_ID) + || request.encryption_algorithm() + != Some(&EncryptionAlgorithmSpec::RsaesOaepSha256) + { + return false; + } + + *plaintext_kek_from_encrypt.lock().unwrap() = request + .plaintext() + .expect("EncryptionManager must provide a plaintext KEK") + .as_ref() + .to_vec(); + true + }) + .then_output(|| { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(b"wrapped-asymmetric-kek")) + .build() + }); + let plaintext_kek_for_decrypt = Arc::clone(&plaintext_kek); + let decrypt_rule = mock!(aws_sdk_kms::Client::decrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::RsaesOaepSha256) + && request.ciphertext_blob().map(AsRef::as_ref) + == Some(b"wrapped-asymmetric-kek".as_slice()) + }) + .then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext_kek_for_decrypt.lock().unwrap().clone())) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&encrypt_rule, &decrypt_rule]); + let kms: Arc = Arc::new(AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::RsaesOaepSha256, + DataKeySpec::Aes256, + )); + let manager = EncryptionManager::builder() + .kms_client(Arc::clone(&kms)) + .table_key_id(KEY_ID) + .build(); + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let encrypted_key_id = manager + .encrypt_manifest_list_key_metadata(&key_metadata) + .await + .unwrap(); + let encryption_keys = manager.with_encryption_keys(Clone::clone); + let reader = EncryptionManager::builder() + .kms_client(kms) + .table_key_id(KEY_ID) + .encryption_keys(encryption_keys) + .build(); + let decrypted = reader + .decrypt_manifest_list_key_metadata(&encrypted_key_id) + .await + .unwrap(); + + assert_eq!(decrypted, key_metadata); + assert_eq!(encrypt_rule.num_calls(), 1); + assert_eq!(decrypt_rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_aws_service_error_messages_name_operation_and_key() { + let not_found = || { + NotFoundException::builder() + .message("test key not found") + .build() + }; + let encrypt_rule = mock!(aws_sdk_kms::Client::encrypt) + .then_error(move || EncryptError::NotFoundException(not_found())); + let decrypt_rule = mock!(aws_sdk_kms::Client::decrypt).then_error(|| { + DecryptError::DisabledException( + DisabledException::builder() + .message("test key disabled") + .build(), + ) + }); + let generate_rule = mock!(aws_sdk_kms::Client::generate_data_key) + .then_error(move || GenerateDataKeyError::NotFoundException(not_found())); + let client = mock_client!(aws_sdk_kms, RuleMode::Sequential, &[ + &encrypt_rule, + &decrypt_rule, + &generate_rule + ]); + let kms = test_client(client); + + let error = kms.wrap_key(b"key", KEY_ID).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(std::error::Error::source(&error).is_some()); + assert_eq!( + error.message(), + format!( + "AWS KMS Encrypt failed for key '{KEY_ID}': NotFoundException: test key not found" + ) + ); + + let error = kms.unwrap_key(b"wrapped", KEY_ID).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!( + error + .message() + .starts_with(&format!("AWS KMS Decrypt failed for key '{KEY_ID}': ")), + "{error}" + ); + assert!(error.message().contains("test key disabled"), "{error}"); + + let error = kms.generate_key(KEY_ID).await.err().unwrap(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!( + error.message().starts_with(&format!( + "AWS KMS GenerateDataKey failed for key '{KEY_ID}': " + )), + "{error}" + ); + assert!(error.message().contains("test key not found"), "{error}"); + } + + #[test] + fn test_request_failure_describes_transport_errors() { + let error = SdkError::::dispatch_failure(ConnectorError::other( + "an error occurred while loading credentials".into(), + None, + )); + + let error = AwsKeyManagementClient::request_failed("Decrypt", KEY_ID, None, error); + + // Nested transport errors are part of the message. + assert!( + error + .message() + .ends_with(": an error occurred while loading credentials"), + "{error}" + ); + } + + #[test] + fn test_request_failure_redacts_plaintext_key() { + let plaintext = b"0123456789abcdef"; + let encoded = BASE64_STANDARD.encode(plaintext); + let error = SdkError::::dispatch_failure(ConnectorError::other( + format!(r#"failed to apply endpoint to request {{"Plaintext":"{encoded}"}}"#).into(), + None, + )); + + let error = + AwsKeyManagementClient::request_failed("Encrypt", KEY_ID, Some(plaintext), error); + + assert!(error.message().contains(""), "{error}"); + for output in [ + error.to_string(), + format!("{error:?}"), + format!("{error:#?}"), + ] { + assert!(!output.contains(&encoded), "{output}"); + } + } + + #[test] + fn test_request_failure_includes_unmodeled_error_code_and_message() { + let error = SdkError::::service_error( + DecryptError::generic( + ErrorMetadata::builder() + .code("AccessDeniedException") + .message("User is not authorized to perform: kms:Decrypt") + .build(), + ), + String::new(), + ); + + let error = AwsKeyManagementClient::request_failed("Decrypt", KEY_ID, None, error); + + assert!(error.message().contains("AccessDeniedException"), "{error}"); + assert!( + error + .message() + .ends_with(": User is not authorized to perform: kms:Decrypt"), + "{error}" + ); + } + + #[test] + fn test_request_failure_drops_raw_response() { + let error = SdkError::service_error( + DecryptError::NotFoundException(NotFoundException::builder().build()), + r#"{"Plaintext":"SECRET"}"#.to_string(), + ); + + let error = AwsKeyManagementClient::request_failed("Decrypt", KEY_ID, None, error); + + let debug = format!("{error:#?}"); + assert!(debug.contains("NotFoundException"), "{debug}"); + assert!(!debug.contains("SECRET"), "{debug}"); + } + + #[tokio::test] + async fn test_reject_missing_or_empty_response_fields() { + let encrypt_missing = + mock!(aws_sdk_kms::Client::encrypt).then_output(|| EncryptOutput::builder().build()); + let encrypt_empty = mock!(aws_sdk_kms::Client::encrypt).then_output(|| { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(Vec::new())) + .build() + }); + let decrypt_missing = + mock!(aws_sdk_kms::Client::decrypt).then_output(|| DecryptOutput::builder().build()); + let generate_missing_plaintext = + mock!(aws_sdk_kms::Client::generate_data_key).then_output(|| { + GenerateDataKeyOutput::builder() + .ciphertext_blob(Blob::new(b"wrapped")) + .build() + }); + let generate_missing_ciphertext = mock!(aws_sdk_kms::Client::generate_data_key) + .then_output(|| { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new([0; 32])) + .build() + }); + let generate_empty_ciphertext = + mock!(aws_sdk_kms::Client::generate_data_key).then_output(|| { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new([0; 32])) + .ciphertext_blob(Blob::new(Vec::new())) + .build() + }); + let client = mock_client!(aws_sdk_kms, RuleMode::Sequential, &[ + &encrypt_missing, + &encrypt_empty, + &decrypt_missing, + &generate_missing_plaintext, + &generate_missing_ciphertext, + &generate_empty_ciphertext + ]); + let kms = test_client(client); + + let errors = [ + kms.wrap_key(b"key", KEY_ID).await.unwrap_err(), + kms.wrap_key(b"key", KEY_ID).await.unwrap_err(), + kms.unwrap_key(b"wrapped", KEY_ID).await.unwrap_err(), + kms.generate_key(KEY_ID).await.err().unwrap(), + kms.generate_key(KEY_ID).await.err().unwrap(), + kms.generate_key(KEY_ID).await.err().unwrap(), + ]; + let expected = [ + "AWS KMS Encrypt response did not contain ciphertext_blob", + "AWS KMS Encrypt response did not contain ciphertext_blob", + "AWS KMS Decrypt response did not contain plaintext", + "AWS KMS GenerateDataKey response did not contain plaintext", + "AWS KMS GenerateDataKey response did not contain ciphertext_blob", + "AWS KMS GenerateDataKey response did not contain ciphertext_blob", + ]; + for (error, expected) in errors.iter().zip(expected) { + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert_eq!(error.message(), expected); + } + } + + #[tokio::test] + async fn test_reject_generated_key_not_matching_key_spec() { + for (data_key_spec, length, expected) in [ + ( + DataKeySpec::Aes256, + 24, + "a 24-byte key for key spec AES_256, expected 32 bytes", + ), + ( + DataKeySpec::Aes128, + 32, + "a 32-byte key for key spec AES_128, expected 16 bytes", + ), + ] { + let rule = mock!(aws_sdk_kms::Client::generate_data_key).then_output(move || { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new(vec![0; length])) + .ciphertext_blob(Blob::new(b"wrapped")) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::SymmetricDefault, + data_key_spec, + ); + + let error = kms.generate_key(KEY_ID).await.err().unwrap(); + + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert_eq!( + error.message(), + format!("AWS KMS GenerateDataKey returned {expected}") + ); + } + } + + #[test] + fn test_request_failure_does_not_repeat_aws_message() { + // Errors parsed from AWS responses carry the message in their metadata + // as well as in the modeled exception. + let error = SdkError::::service_error( + EncryptError::NotFoundException( + NotFoundException::builder() + .message("test key not found") + .meta( + ErrorMetadata::builder() + .code("NotFoundException") + .message("test key not found") + .build(), + ) + .build(), + ), + String::new(), + ); + + let error = AwsKeyManagementClient::request_failed("Encrypt", KEY_ID, None, error); + + assert_eq!( + error.message().matches("test key not found").count(), + 1, + "{error}" + ); + } + + #[tokio::test] + async fn test_symmetric_manager_roundtrip_uses_generate_data_key() { + let plaintext_kek = [7; 32]; + let wrapped_kek = b"wrapped-symmetric-kek"; + let generate_rule = mock!(aws_sdk_kms::Client::generate_data_key) + .match_requests(|request| request.key_id() == Some(KEY_ID)) + .then_output(move || { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new(plaintext_kek)) + .ciphertext_blob(Blob::new(wrapped_kek)) + .build() + }); + let decrypt_rule = mock!(aws_sdk_kms::Client::decrypt) + .match_requests(move |request| { + request.key_id() == Some(KEY_ID) + && request.ciphertext_blob().map(AsRef::as_ref) == Some(wrapped_kek.as_slice()) + }) + .then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext_kek)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&generate_rule, &decrypt_rule]); + let kms: Arc = Arc::new(test_client(client)); + let manager = EncryptionManager::builder() + .kms_client(Arc::clone(&kms)) + .table_key_id(KEY_ID) + .build(); + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let encrypted_key_id = manager + .encrypt_manifest_list_key_metadata(&key_metadata) + .await + .unwrap(); + let encryption_keys = manager.with_encryption_keys(Clone::clone); + let reader = EncryptionManager::builder() + .kms_client(kms) + .table_key_id(KEY_ID) + .encryption_keys(encryption_keys) + .build(); + let decrypted = reader + .decrypt_manifest_list_key_metadata(&encrypted_key_id) + .await + .unwrap(); + + assert_eq!(decrypted, key_metadata); + assert_eq!(generate_rule.num_calls(), 1); + assert_eq!(decrypt_rule.num_calls(), 1); + } + + fn test_client(client: aws_sdk_kms::Client) -> AwsKeyManagementClient { + AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::SymmetricDefault, + DataKeySpec::Aes256, + ) + } +} diff --git a/crates/kms/src/aws/config.rs b/crates/kms/src/aws/config.rs new file mode 100644 index 0000000000..cbcaeb782e --- /dev/null +++ b/crates/kms/src/aws/config.rs @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use aws_sdk_kms::config::Credentials; +use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; +use http::Uri; +use http::uri::Scheme; +use iceberg::io::{CLIENT_REGION, S3_ASSUME_ROLE_ARN}; +use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; + +/// Catalog property overriding the AWS KMS service endpoint. +pub const KMS_ENDPOINT: &str = "kms.endpoint"; +/// Catalog property selecting the AWS KMS encryption algorithm. +pub const KMS_ENCRYPTION_ALGORITHM_SPEC: &str = "kms.encryption-algorithm-spec"; +/// Default AWS KMS encryption algorithm, matching Iceberg Java. +pub const KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str = "SYMMETRIC_DEFAULT"; +/// Catalog property selecting the size of keys generated by AWS KMS. +pub const KMS_DATA_KEY_SPEC: &str = "kms.data-key-spec"; +/// Default AWS KMS generated key size, matching Iceberg Java. +pub const KMS_DATA_KEY_SPEC_DEFAULT: &str = "AES_256"; + +// Properties shared with the Glue and S3 Tables catalogs. +pub(crate) const AWS_PROFILE_NAME: &str = "profile_name"; +pub(crate) const AWS_REGION_NAME: &str = "region_name"; +pub(crate) const AWS_ACCESS_KEY_ID: &str = "aws_access_key_id"; +pub(crate) const AWS_SECRET_ACCESS_KEY: &str = "aws_secret_access_key"; +pub(crate) const AWS_SESSION_TOKEN: &str = "aws_session_token"; + +#[derive(Properties)] +pub(crate) struct AwsKmsConfig { + #[property(key = KMS_ENDPOINT, default = None, parse_with = parse_endpoint)] + pub(crate) endpoint: Option, + #[property( + key = KMS_ENCRYPTION_ALGORITHM_SPEC, + default = KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT, + parse_with = parse_encryption_algorithm + )] + pub(crate) encryption_algorithm: EncryptionAlgorithmSpec, + #[property( + key = KMS_DATA_KEY_SPEC, + default = KMS_DATA_KEY_SPEC_DEFAULT, + parse_with = parse_data_key_spec + )] + pub(crate) data_key_spec: DataKeySpec, +} + +/// Catalog properties used to build the AWS SDK configuration when the +/// application does not supply one. +#[derive(Properties)] +pub(crate) struct AwsSdkProperties { + #[property( + key = CLIENT_REGION, + additional_keys = [AWS_REGION_NAME], + default = None, + parse_properties_with = parse_first_present + )] + pub(crate) region: Option, + #[property(key = AWS_PROFILE_NAME, default = None)] + pub(crate) profile_name: Option, + #[property(key = AWS_ACCESS_KEY_ID, default = None)] + access_key_id: Option, + #[property(key = AWS_SECRET_ACCESS_KEY, default = None)] + secret_access_key: Option, + #[property(key = AWS_SESSION_TOKEN, default = None)] + session_token: Option, + /// Not supported yet; rejected rather than ignored, so that the KMS client + /// never silently uses a different identity than the catalog's file IO. + #[property(key = S3_ASSUME_ROLE_ARN, default = None)] + pub(crate) assume_role_arn: Option, +} + +impl AwsSdkProperties { + /// Static credentials from the catalog properties, if any are set. + pub(crate) fn static_credentials(&self) -> Result> { + match ( + &self.access_key_id, + &self.secret_access_key, + &self.session_token, + ) { + (Some(access_key_id), Some(secret_access_key), session_token) => { + Ok(Some(Credentials::new( + access_key_id, + secret_access_key, + session_token.clone(), + None, + "catalog-properties", + ))) + } + (None, None, None) => Ok(None), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "AWS static credentials require both '{AWS_ACCESS_KEY_ID}' and '{AWS_SECRET_ACCESS_KEY}'" + ), + )), + } + } +} + +/// Accepts the endpoints the AWS SDK can apply to a request: `http` or `https` +/// URIs with a host. +fn parse_endpoint(value: &str) -> Result { + let invalid = || { + Error::new( + ErrorKind::DataInvalid, + format!("AWS KMS endpoint must be an http or https URL with a host: '{value}'"), + ) + }; + let uri = value + .parse::() + .map_err(|source| invalid().with_source(source))?; + let has_supported_scheme = uri + .scheme() + .is_some_and(|scheme| *scheme == Scheme::HTTP || *scheme == Scheme::HTTPS); + let has_host = uri.host().is_some_and(|host| !host.is_empty()); + if has_supported_scheme && has_host { + Ok(value.to_string()) + } else { + Err(invalid()) + } +} + +fn parse_encryption_algorithm(value: &str) -> Result { + EncryptionAlgorithmSpec::try_parse(value).map_err(|source| { + Error::new( + ErrorKind::DataInvalid, + format!("Unsupported AWS KMS encryption algorithm: {value}"), + ) + .with_source(source) + }) +} + +fn parse_data_key_spec(value: &str) -> Result { + DataKeySpec::try_parse(value).map_err(|source| { + Error::new( + ErrorKind::DataInvalid, + format!("Unsupported AWS KMS data key spec: {value}"), + ) + .with_source(source) + }) +} + +/// Uses the value of the first key present, so Iceberg Java's key takes +/// precedence over the Glue-compatible alias. +fn parse_first_present( + properties: &HashMap, + key: &str, + additional_keys: &[&str], + default: Option, +) -> Result> { + Ok(std::iter::once(key) + .chain(additional_keys.iter().copied()) + .find_map(|key| properties.get(key).cloned()) + .or(default)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_defaults_match_iceberg_java() { + let config = AwsKmsConfig::from_properties(&HashMap::new()).unwrap(); + + assert_eq!( + config.encryption_algorithm, + EncryptionAlgorithmSpec::SymmetricDefault + ); + assert_eq!(config.data_key_spec, DataKeySpec::Aes256); + assert_eq!(config.endpoint, None); + } + + #[test] + fn test_parse_overrides() { + let properties = HashMap::from([ + ( + KMS_ENDPOINT.to_string(), + "http://localhost:4566".to_string(), + ), + ( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "RSAES_OAEP_SHA_256".to_string(), + ), + (KMS_DATA_KEY_SPEC.to_string(), "AES_128".to_string()), + ]); + + let config = AwsKmsConfig::from_properties(&properties).unwrap(); + assert_eq!(config.endpoint.as_deref(), Some("http://localhost:4566")); + assert_eq!( + config.encryption_algorithm, + EncryptionAlgorithmSpec::RsaesOaepSha256 + ); + assert_eq!(config.data_key_spec, DataKeySpec::Aes128); + } + + #[test] + fn test_reject_incomplete_static_credentials() { + for keys in [ + vec![AWS_ACCESS_KEY_ID], + vec![AWS_SECRET_ACCESS_KEY], + vec![AWS_SESSION_TOKEN], + vec![AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN], + ] { + let properties = keys + .iter() + .map(|key| (key.to_string(), "value".to_string())) + .collect(); + let sdk_properties = AwsSdkProperties::from_properties(&properties).unwrap(); + let Err(error) = sdk_properties.static_credentials() else { + panic!("incomplete credentials {keys:?} must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + } + + #[test] + fn test_endpoint_validation() { + for endpoint in [ + "http://localhost:4566", + "http://[::1]:4566", + "https://kms.eu-west-2.amazonaws.com", + "https://kms-fips.us-east-1.amazonaws.com/", + "HTTPS://kms.eu-west-2.amazonaws.com", + "http://localhost:4566/kms", + ] { + assert_eq!(parse_endpoint(endpoint).unwrap(), endpoint); + } + // Unparsable URIs make the SDK describe the request, body included, in + // its error. + for endpoint in [ + "", + "localhost:4566", + "http://", + "https:///path", + "http://a b", + "http://:4566", + "http://%zz", + "http://\u{fc}.example", + "ftp://host", + ] { + assert_eq!( + parse_endpoint(endpoint).unwrap_err().kind(), + ErrorKind::DataInvalid, + "{endpoint}" + ); + } + } + + #[test] + fn test_reject_unknown_values() { + let bad_algorithm = HashMap::from([( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "unknown".to_string(), + )]); + let Err(error) = AwsKmsConfig::from_properties(&bad_algorithm) else { + panic!("unknown encryption algorithms must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + + let bad_key_spec = HashMap::from([(KMS_DATA_KEY_SPEC.to_string(), "AES_192".to_string())]); + let Err(error) = AwsKmsConfig::from_properties(&bad_key_spec) else { + panic!("unknown data-key specifications must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/kms/src/aws/factory.rs b/crates/kms/src/aws/factory.rs new file mode 100644 index 0000000000..4ea003c302 --- /dev/null +++ b/crates/kms/src/aws/factory.rs @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use async_trait::async_trait; +use aws_config::{BehaviorVersion, ConfigLoader, Region, SdkConfig}; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::io::S3_ASSUME_ROLE_ARN; +use iceberg::{Error, ErrorKind, Result}; + +use super::client::AwsKeyManagementClient; +use super::config::{AwsKmsConfig, AwsSdkProperties}; + +/// Factory for AWS KMS-backed Iceberg key management clients. +/// +/// By default, AWS credentials and region are resolved through the AWS SDK's +/// standard provider chains. Catalog properties can override the region, +/// profile, or static credentials. Applications with custom credential or HTTP +/// providers can supply a complete [`SdkConfig`] through +/// [`with_sdk_config`](Self::with_sdk_config). +#[derive(Clone, Default)] +pub struct AwsKmsClientFactory { + sdk_config: Option, +} + +impl AwsKmsClientFactory { + /// Create a factory using catalog properties and the AWS SDK default chains. + pub fn new() -> Self { + Self::default() + } + + /// Use an application-provided AWS SDK configuration. + /// + /// When set, the region, profile, and credential catalog properties are not + /// read. KMS-specific properties such as `kms.endpoint` continue to be + /// applied. + pub fn with_sdk_config(mut self, sdk_config: SdkConfig) -> Self { + self.sdk_config = Some(sdk_config); + self + } +} + +impl fmt::Debug for AwsKmsClientFactory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AwsKmsClientFactory") + .field("has_sdk_config", &self.sdk_config.is_some()) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl KmsClientFactory for AwsKmsClientFactory { + async fn create_kms_client( + &self, + properties: &HashMap, + ) -> Result> { + let kms_config = AwsKmsConfig::from_properties(properties)?; + let sdk_config = match &self.sdk_config { + Some(config) => config.clone(), + None => { + let sdk_properties = AwsSdkProperties::from_properties(properties)?; + create_sdk_config( + &sdk_properties, + aws_config::defaults(BehaviorVersion::latest()), + ) + .await? + } + }; + + let mut builder = aws_sdk_kms::config::Builder::from(&sdk_config); + if sdk_config.behavior_version().is_none() { + builder = builder.behavior_version(BehaviorVersion::latest()); + } + if let Some(endpoint) = &kms_config.endpoint { + builder = builder.endpoint_url(endpoint); + } + + Ok(Arc::new(AwsKeyManagementClient::new( + aws_sdk_kms::Client::from_conf(builder.build()), + kms_config.encryption_algorithm, + kms_config.data_key_spec, + ))) + } +} + +async fn create_sdk_config( + properties: &AwsSdkProperties, + mut loader: ConfigLoader, +) -> Result { + if properties.assume_role_arn.is_some() { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "'{S3_ASSUME_ROLE_ARN}' is not supported by the AWS KMS client yet; supply an SdkConfig with the desired credentials through AwsKmsClientFactory::with_sdk_config" + ), + )); + } + if let Some(credentials) = properties.static_credentials()? { + loader = loader.credentials_provider(credentials); + } + if let Some(profile) = &properties.profile_name { + loader = loader.profile_name(profile); + } + if let Some(region) = &properties.region { + loader = loader.region(Region::new(region.clone())); + } + Ok(loader.load().await) +} + +#[cfg(test)] +mod tests { + use aws_sdk_kms::config::{Credentials, ProvideCredentials}; + use aws_smithy_http_client::test_util::{CaptureRequestReceiver, capture_request}; + use base64::Engine as _; + use base64::prelude::BASE64_STANDARD; + use iceberg::io::CLIENT_REGION; + + use super::*; + use crate::aws::config::{ + AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + }; + use crate::aws::{KMS_DATA_KEY_SPEC, KMS_ENCRYPTION_ALGORITHM_SPEC, KMS_ENDPOINT}; + + const ROLE_ARN: &str = "arn:aws:iam::123456789012:role/kms"; + + /// A config loader that ignores the host's `AWS_*` environment variables + /// and shared configuration files. + fn test_loader() -> ConfigLoader { + aws_config::defaults(BehaviorVersion::latest()).empty_test_environment() + } + + /// A fully loaded SDK configuration whose HTTP client captures the first + /// request instead of sending it. + async fn capturing_sdk_config() -> (SdkConfig, CaptureRequestReceiver) { + let (http_client, request) = capture_request(None); + let sdk_config = test_loader() + .region(Region::new("eu-west-2")) + .credentials_provider(Credentials::new("access", "secret", None, None, "test")) + .http_client(http_client) + .load() + .await; + (sdk_config, request) + } + + async fn capturing_factory() -> (AwsKmsClientFactory, CaptureRequestReceiver) { + let (sdk_config, request) = capturing_sdk_config().await; + ( + AwsKmsClientFactory::new().with_sdk_config(sdk_config), + request, + ) + } + + fn request_body(request: CaptureRequestReceiver) -> String { + let request = request.expect_request(); + String::from_utf8(request.body().bytes().unwrap().to_vec()).unwrap() + } + + fn sdk_properties(properties: &[(&str, &str)]) -> AwsSdkProperties { + let properties = properties + .iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect(); + AwsSdkProperties::from_properties(&properties).unwrap() + } + + #[tokio::test] + async fn test_kms_endpoint_overrides_injected_sdk_config() { + let (factory, request) = capturing_factory().await; + let properties = HashMap::from([( + KMS_ENDPOINT.to_string(), + "http://localhost:4566".to_string(), + )]); + + let client = factory.create_kms_client(&properties).await.unwrap(); + // The captured response is empty, so only the outgoing request matters here. + let _ = client.unwrap_key(b"wrapped", "test-key").await; + + assert_eq!(request.expect_request().uri(), "http://localhost:4566/"); + } + + #[tokio::test] + async fn test_reject_invalid_kms_endpoint() { + let factory = AwsKmsClientFactory::new().with_sdk_config(capturing_sdk_config().await.0); + let properties = HashMap::from([(KMS_ENDPOINT.to_string(), "localhost:4566".to_string())]); + + let Err(error) = factory.create_kms_client(&properties).await else { + panic!("endpoints without a scheme must be rejected"); + }; + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_factory_applies_kms_data_key_spec() { + let (factory, request) = capturing_factory().await; + let properties = HashMap::from([(KMS_DATA_KEY_SPEC.to_string(), "AES_128".to_string())]); + + let client = factory.create_kms_client(&properties).await.unwrap(); + let _ = client.generate_key("test-key").await; + + assert!(request_body(request).contains(r#""KeySpec":"AES_128""#)); + } + + #[tokio::test] + async fn test_factory_applies_kms_encryption_algorithm_spec() { + let (factory, request) = capturing_factory().await; + let properties = HashMap::from([( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "RSAES_OAEP_SHA_256".to_string(), + )]); + + let client = factory.create_kms_client(&properties).await.unwrap(); + let _ = client.wrap_key(b"key", "test-key").await; + + assert!(request_body(request).contains(r#""EncryptionAlgorithm":"RSAES_OAEP_SHA_256""#)); + } + + #[tokio::test] + async fn test_create_client_adds_missing_behavior_version() { + let factory = AwsKmsClientFactory::new().with_sdk_config(SdkConfig::builder().build()); + + // `aws_sdk_kms::Client::from_conf` panics if no behavior version is set. + factory.create_kms_client(&HashMap::new()).await.unwrap(); + } + + #[tokio::test] + async fn test_static_credentials_and_region_are_applied() { + let properties = sdk_properties(&[ + (AWS_ACCESS_KEY_ID, "access"), + (AWS_SECRET_ACCESS_KEY, "secret"), + (AWS_SESSION_TOKEN, "token"), + (AWS_REGION_NAME, "eu-west-2"), + ]); + + let config = create_sdk_config(&properties, test_loader()).await.unwrap(); + let credentials = config + .credentials_provider() + .expect("static credentials must install a credentials provider") + .provide_credentials() + .await + .unwrap(); + + assert_eq!(config.region().map(Region::as_ref), Some("eu-west-2")); + assert_eq!(credentials.access_key_id(), "access"); + assert_eq!(credentials.secret_access_key(), "secret"); + assert_eq!(credentials.session_token(), Some("token")); + } + + #[tokio::test] + async fn test_client_region_takes_precedence_over_region_name() { + let properties = + sdk_properties(&[(CLIENT_REGION, "eu-west-1"), (AWS_REGION_NAME, "eu-west-2")]); + + let config = create_sdk_config(&properties, test_loader()).await.unwrap(); + + assert_eq!(config.region().map(Region::as_ref), Some("eu-west-1")); + } + + #[tokio::test] + async fn test_reject_assume_role() { + // Rejected before the AWS environment is read. + let properties = HashMap::from([(S3_ASSUME_ROLE_ARN.to_string(), ROLE_ARN.to_string())]); + + let Err(error) = AwsKmsClientFactory::new() + .create_kms_client(&properties) + .await + else { + panic!("assume-role properties must be rejected"); + }; + + assert_eq!(error.kind(), ErrorKind::FeatureUnsupported); + } + + #[tokio::test] + async fn test_reject_incomplete_static_credentials() { + // Rejected before the AWS environment is read. + let properties = HashMap::from([(AWS_ACCESS_KEY_ID.to_string(), "access".to_string())]); + + let Err(error) = AwsKmsClientFactory::new() + .create_kms_client(&properties) + .await + else { + panic!("incomplete static credentials must be rejected"); + }; + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_injected_sdk_config_ignores_sdk_properties() { + let factory = AwsKmsClientFactory::new().with_sdk_config(capturing_sdk_config().await.0); + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "access".to_string()), + (S3_ASSUME_ROLE_ARN.to_string(), ROLE_ARN.to_string()), + ]); + + assert!(factory.create_kms_client(&properties).await.is_ok()); + } + + #[tokio::test] + async fn test_invalid_sdk_endpoint_does_not_leak_plaintext_key() { + // The SDK describes the whole request, body included, when it cannot + // apply an endpoint URL. An injected `SdkConfig` is not validated. + let (http_client, _request) = capture_request(None); + let sdk_config = test_loader() + .region(Region::new("eu-west-2")) + .credentials_provider(Credentials::new("access", "secret", None, None, "test")) + .endpoint_url("http://%zz") + .http_client(http_client) + .load() + .await; + let factory = AwsKmsClientFactory::new().with_sdk_config(sdk_config); + let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + let plaintext = b"0123456789abcdef"; + let encoded = BASE64_STANDARD.encode(plaintext); + + let error = client.wrap_key(plaintext, "test-key").await.unwrap_err(); + + // Guards against this passing without the SDK describing the request. + assert!(error.message().contains(""), "{error}"); + + for output in [ + error.to_string(), + format!("{error:?}"), + format!("{error:#?}"), + ] { + assert!(!output.contains(&encoded), "{output}"); + } + } +} diff --git a/crates/kms/src/aws/mod.rs b/crates/kms/src/aws/mod.rs new file mode 100644 index 0000000000..3edf18f319 --- /dev/null +++ b/crates/kms/src/aws/mod.rs @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS Key Management Service integration. +//! +//! This module provides [`AwsKmsClientFactory`], +//! which creates AWS-backed implementations of Iceberg's +//! [`KeyManagementClient`](iceberg::encryption::KeyManagementClient). The factory can +//! be supplied to any Iceberg catalog builder through +//! [`CatalogBuilder::with_kms_client_factory`](iceberg::CatalogBuilder::with_kms_client_factory). +//! +//! # Permissions +//! +//! The KMS key referenced by a table's `encryption.key-id` property requires +//! `kms:Decrypt` for reads. Writes with `SYMMETRIC_DEFAULT` use +//! `kms:GenerateDataKey`. Writes with an asymmetric encryption algorithm +//! generate the key-encryption key locally and require `kms:Encrypt` to wrap it. +//! +//! Iceberg Java always generates key-encryption keys locally and wraps them +//! with `kms:Encrypt`, so key policies written for Java writers must also grant +//! `kms:GenerateDataKey` to Rust writers using `SYMMETRIC_DEFAULT`. Tables +//! written by either implementation are readable by the other. +//! +//! # Configuration +//! +//! The factory reads the catalog properties supplied by the application when +//! the catalog is constructed. Configuration returned later by a REST catalog +//! server is not currently available when the KMS client is created. +//! +//! These KMS properties match Iceberg Java: +//! +//! - `kms.endpoint` overrides the KMS service endpoint. It must be an `http` or +//! `https` URL. +//! - `kms.encryption-algorithm-spec` selects the KMS encryption algorithm and +//! defaults to `SYMMETRIC_DEFAULT`. +//! - `kms.data-key-spec` selects the size of key-encryption keys generated by +//! KMS and defaults to `AES_256`. It only applies to `SYMMETRIC_DEFAULT`; +//! with an asymmetric algorithm, locally generated key-encryption keys are +//! sized by the table property `encryption.data-key-length`, like data keys. +//! +//! The AWS SDK's default credential and region provider chains are used, and +//! the properties below override them. They are not read when an +//! [`SdkConfig`](aws_config::SdkConfig) is supplied through +//! [`AwsKmsClientFactory::with_sdk_config`]. +//! +//! - `client.region` selects the region, as in Iceberg Java. The `region_name` +//! property used by the Glue and S3 Tables catalogs is also accepted, with +//! lower precedence. +//! - `profile_name` selects a profile from the shared AWS configuration files. +//! - `aws_access_key_id`, `aws_secret_access_key`, and the optional +//! `aws_session_token` provide static credentials. +//! +//! Assuming an IAM role through `client.assume-role.*` is not supported yet: +//! catalog properties that set `client.assume-role.arn` are rejected, rather +//! than having the KMS client use a different identity than the catalog's S3 +//! file IO. Supply an [`SdkConfig`](aws_config::SdkConfig) with the desired +//! credentials instead. +//! +//! `client.credentials-provider` and `client.factory` name Java classes and +//! are ignored, as are the `s3.*` credentials used by S3 file IO. +//! +//! # Security considerations +//! +//! With an asymmetric encryption algorithm, wrapping a key-encryption key only +//! requires the KMS public key. Anyone who can write table metadata can +//! therefore add a key-encryption key they know, without any KMS permissions; +//! `SYMMETRIC_DEFAULT` requires `kms:GenerateDataKey` or `kms:Encrypt` instead. +//! +//! With an asymmetric encryption algorithm, plaintext key-encryption keys are +//! sent to KMS, and the AWS SDK's `TRACE` logs include request bodies. +//! +//! # Example +//! +//! ```rust,no_run +//! use std::collections::HashMap; +//! use std::sync::Arc; +//! +//! use iceberg::CatalogBuilder; +//! use iceberg::io::CLIENT_REGION; +//! use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +//! use iceberg_kms::aws::AwsKmsClientFactory; +//! +//! # async fn example() -> iceberg::Result<()> { +//! let properties = HashMap::from([ +//! ( +//! MEMORY_CATALOG_WAREHOUSE.to_string(), +//! "s3://bucket/warehouse".to_string(), +//! ), +//! (CLIENT_REGION.to_string(), "eu-west-2".to_string()), +//! ]); +//! // Any catalog builder accepts a KMS client factory. A storage factory for +//! // the warehouse is omitted for brevity. +//! let catalog = MemoryCatalogBuilder::default() +//! .with_kms_client_factory(Arc::new(AwsKmsClientFactory::new())) +//! .load("catalog", properties) +//! .await?; +//! # let _ = catalog; +//! # Ok(()) +//! # } +//! ``` + +mod client; +mod config; +mod factory; + +pub use config::{ + KMS_DATA_KEY_SPEC, KMS_DATA_KEY_SPEC_DEFAULT, KMS_ENCRYPTION_ALGORITHM_SPEC, + KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT, KMS_ENDPOINT, +}; +pub use factory::AwsKmsClientFactory; diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs new file mode 100644 index 0000000000..7269472f8a --- /dev/null +++ b/crates/kms/src/lib.rs @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key management service integrations for Apache Iceberg client-side encryption. +//! +//! Provider implementations are enabled through crate features. Enable `aws` +//! to use the `aws` module. + +#![deny(missing_docs)] + +#[cfg(feature = "aws")] +pub mod aws;