diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 176d43abf6..cd60d23475 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -37,4 +37,18 @@ ignore = [ # once the transitive dependency set permits quick-xml >=0.41.0. "RUSTSEC-2026-0194", "RUSTSEC-2026-0195", + # `h2` 0.3.27 accepts an unbounded number of empty DATA frames (low-severity + # DoS). + # + # It is present only through `aws-sdk-kms/test-util` -> + # `aws-smithy-runtime/legacy-test-util`, which is required by the official + # `aws-smithy-mocks` test framework, and is absent from normal/build + # dependencies. + # + # Per SECURITY-THREAT-MODEL.md, malformed-input DoS is hardening rather than + # an Iceberg Rust trust-boundary vulnerability. + # + # Remove after the fix in + # https://github.com/smithy-lang/smithy-rs/pull/4827 reaches an AWS SDK release. + "RUSTSEC-2026-0258", ] diff --git a/Cargo.lock b/Cargo.lock index 64c5c61875..6359342cd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -534,9 +534,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.18" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33f815b73a3899c03b380d543532e5865f230dce9678d108dc10732a8682275" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -544,11 +544,11 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.1", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema 0.1.0", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", @@ -565,9 +565,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -600,14 +600,14 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.7.5" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9b9de216a988dd54b754a82a7660cfe14cee4f6782ae4524470972fa0ccb39" +checksum = "25b43ad47adc2517efe3d706559d94b97e50e80e0321b3cadbc9f77cee88adcd" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", - "aws-smithy-http 0.63.6", + "aws-smithy-http 0.64.0", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -634,7 +634,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-json 0.62.7", "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -648,6 +648,32 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-kms" +version = "1.120.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b0fe38fee2ba5b6cd24d32d08365b314ae1adea123649e061a7eb6300b6f5b" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.5.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-s3tables" version = "1.59.0" @@ -659,7 +685,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http 0.63.6", - "aws-smithy-json", + "aws-smithy-json 0.62.7", "aws-smithy-observability 0.2.6", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -675,19 +701,20 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.102.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c82b3ac19f1431854f7ace3a7531674633e286bfdde21976893bfee36fd493b" +checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", @@ -700,19 +727,20 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.104.0" +version = "1.113.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321000d2b4c5519ee573f73167f612efd7329322d9b26969ad1979f0427f1913" +checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-types", "bytes", @@ -725,20 +753,21 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.107.0" +version = "1.116.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0d328ba962af23ecfa3c9f23b98d3d35e325fa218d7f13d17a6bf522f8a560" +checksum = "035c7392e751d20bf1060d625d023587eb14099646110a9cff7c697bfe4b253a" dependencies = [ "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.63.6", - "aws-smithy-json", - "aws-smithy-observability 0.2.6", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.1", + "aws-smithy-observability 0.3.0", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -751,12 +780,12 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.4.5" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae38512beae0ffee7010fc24e7a8a123c53efdfef42a61e80fda4882418dc71" +checksum = "31d955e76ff96acd555bf06fa0fa6d5bf9335fa84ae7c64481b20ae61d231f70" dependencies = [ "aws-credential-types", - "aws-smithy-http 0.63.6", + "aws-smithy-http 0.64.0", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -826,22 +855,32 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.2.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" +checksum = "7bd25384a4e437aa8d8f339afad4b69e786b936a7cb10db668a7aaf66717b1a8" dependencies = [ "aws-smithy-async", + "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", - "h2", + "bytes", + "h2 0.3.27", + "h2 0.4.16", + "http 0.2.12", "http 1.5.0", - "hyper", + "http-body 0.4.6", + "http-body 1.0.1", + "hyper 0.14.32", + "hyper 1.10.1", "hyper-rustls", "hyper-util", + "indexmap 2.14.0", "pin-project-lite", "rustls", "rustls-native-certs", "rustls-pki-types", + "serde", + "serde_json", "tokio", "tokio-rustls", "tower", @@ -859,6 +898,29 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-mocks" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0922f8ee1ce1f90f0baf9c9036f566161547a7ab9f09571d61d56455ba35598" +dependencies = [ + "aws-smithy-http-client", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + [[package]] name = "aws-smithy-observability" version = "0.2.6" @@ -877,28 +939,50 @@ dependencies = [ "aws-smithy-runtime-api", ] +[[package]] +name = "aws-smithy-protocol-test" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a" +dependencies = [ + "assert-json-diff", + "aws-smithy-runtime-api", + "base64-simd", + "cbor-diag", + "ciborium", + "http 0.2.12", + "pretty_assertions", + "regex-lite", + "roxmltree", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "aws-smithy-query" -version = "0.60.15" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.12.0" +version = "1.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" +checksum = "3296253d3a91b3f938a3f2bcce4daebadcb4aa4228153fcec90f9d23532b4484" dependencies = [ "aws-smithy-async", "aws-smithy-http 0.64.0", "aws-smithy-http-client", "aws-smithy-observability 0.3.0", "aws-smithy-runtime-api", - "aws-smithy-schema 0.2.0", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "bytes", "fastrand", @@ -911,13 +995,14 @@ dependencies = [ "pin-utils", "tokio", "tracing", + "tracing-subscriber", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.13.0" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" +checksum = "6d881a7b7ad179fd6611680c9de89f716fb00ab40299a9a7b8c6913e8f7511a8" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -955,9 +1040,9 @@ dependencies = [ [[package]] name = "aws-smithy-schema" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -966,9 +1051,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.1" +version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" +checksum = "0b791f3ac597193fe1d08b82366986eb1f5bc31f2ac6c194c0855276116c76cd" dependencies = [ "base64-simd", "bytes", @@ -992,23 +1077,26 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.15" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.2.1", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.16" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema 0.1.0", + "aws-smithy-schema 0.2.1", "aws-smithy-types", "rustc_version", "tracing", @@ -1265,6 +1353,25 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbor-diag" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" +dependencies = [ + "bs58", + "chrono", + "data-encoding", + "half", + "nom", + "num-bigint 0.4.8", + "num-rational", + "num-traits", + "separator", + "url", + "uuid", +] + [[package]] name = "cc" version = "1.2.66" @@ -1330,6 +1437,33 @@ dependencies = [ "phf", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -1776,6 +1910,12 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + [[package]] name = "defmt" version = "1.1.1" @@ -2367,6 +2507,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.16" @@ -2607,6 +2766,30 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.10.1" @@ -2617,7 +2800,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", + "h2 0.4.16", "http 1.5.0", "http-body 1.0.1", "httparse", @@ -2636,7 +2819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.5.0", - "hyper", + "hyper 1.10.1", "hyper-util", "rustls", "rustls-native-certs", @@ -2657,12 +2840,12 @@ dependencies = [ "futures-util", "http 1.5.0", "http-body 1.0.1", - "hyper", + "hyper 1.10.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -2912,6 +3095,20 @@ dependencies = [ "uuid", ] +[[package]] +name = "iceberg-kms" +version = "0.10.1" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-kms", + "aws-smithy-mocks", + "iceberg", + "iceberg-catalog-rest", + "iceberg-property-macro", + "tokio", +] + [[package]] name = "iceberg-property-macro" version = "0.10.1" @@ -3561,6 +3758,12 @@ dependencies = [ "serde", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -3621,7 +3824,7 @@ dependencies = [ "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-util", "log", "pin-project-lite", @@ -3727,6 +3930,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "ntapi" version = "0.4.3" @@ -3816,6 +4029,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -4637,7 +4861,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -4676,7 +4900,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -5043,7 +5267,7 @@ dependencies = [ "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-util", "js-sys", "log", @@ -5078,7 +5302,7 @@ dependencies = [ "http 1.5.0", "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-rustls", "hyper-util", "js-sys", @@ -5172,6 +5396,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "roxmltree" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" +dependencies = [ + "xmlparser", +] + [[package]] name = "rsa" version = "0.9.10" @@ -5454,6 +5687,12 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +[[package]] +name = "separator" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" + [[package]] name = "seq-macro" version = "0.3.6" @@ -5530,6 +5769,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -7389,7 +7629,7 @@ dependencies = [ "crc32fast", "futures", "http 1.5.0", - "hyper", + "hyper 1.10.1", "lazy_static", "more-asserts", "rand 0.10.2", diff --git a/Cargo.toml b/Cargo.toml index 661c56b094..29a8eec366 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "crates/iceberg", "crates/integration_tests", "crates/integrations/*", + "crates/kms", "crates/property-macro", "crates/storage/*", "crates/test_utils", @@ -59,11 +60,15 @@ arrow-select = "59.2" arrow-string = "59.2" as-any = "0.3.2" async-trait = "0.1.89" -aws-config = "1.8.7" +aws-config = "1.12" aws-sdk-glue = { version = "1.85", default-features = false, features = [ "default-https-client", "rt-tokio", ] } +aws-sdk-kms = { version = "1.118", default-features = false, features = [ + "default-https-client", + "rt-tokio", +] } aws-sdk-s3tables = { version = "1.28", default-features = false, features = [ "default-https-client", "rt-tokio", @@ -94,6 +99,7 @@ iceberg-catalog-hms = { version = "0.10.0", path = "./crates/catalog/hms" } iceberg-catalog-rest = { version = "0.10.0", path = "./crates/catalog/rest" } iceberg-catalog-s3tables = { version = "0.10.0", path = "./crates/catalog/s3tables" } iceberg-catalog-sql = { version = "0.10.0", path = "./crates/catalog/sql" } +iceberg-kms = { version = "0.10.0", path = "./crates/kms" } iceberg-property-macro = { version = "0.10.0", path = "./crates/property-macro" } iceberg-storage-opendal = { version = "0.10.0", path = "./crates/storage/opendal" } itertools = "0.13" diff --git a/README.md b/README.md index fa974dc767..14d63b3763 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ The Apache Iceberg Rust project is composed of the following components: | [iceberg-catalog-s3tables] | [![iceberg-catalog-s3tables image]][iceberg-catalog-s3tables link] | [![docs release]][iceberg-catalog-s3tables release docs] [![docs dev]][iceberg-catalog-s3tables dev docs] | | [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | | [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | +| [iceberg-kms] | [![iceberg-kms image]][iceberg-kms link] | [![docs release]][iceberg-kms release docs] [![docs dev]][iceberg-kms dev docs] | | [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue @@ -47,6 +48,12 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg release docs]: https://docs.rs/iceberg [iceberg dev docs]: https://rust.iceberg.apache.org/api/iceberg/ +[iceberg-kms]: crates/kms/README.md +[iceberg-kms image]: https://img.shields.io/crates/v/iceberg-kms.svg +[iceberg-kms link]: https://crates.io/crates/iceberg-kms +[iceberg-kms release docs]: https://docs.rs/iceberg-kms +[iceberg-kms dev docs]: https://rust.iceberg.apache.org/api/iceberg_kms/ + [iceberg-catalog-glue]: crates/catalog/glue/README.md [iceberg-catalog-glue image]: https://img.shields.io/crates/v/iceberg-catalog-glue.svg [iceberg-catalog-glue link]: https://crates.io/crates/iceberg-catalog-glue diff --git a/crates/kms/Cargo.toml b/crates/kms/Cargo.toml new file mode 100644 index 0000000000..188904dcb2 --- /dev/null +++ b/crates/kms/Cargo.toml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-kms" +publish = true +readme = "README.md" +rust-version = { workspace = true } +version = { workspace = true } + +categories = ["database"] +description = "Key management service integrations for Apache Iceberg client-side encryption" +keywords = ["iceberg", "kms", "encryption"] +license = { workspace = true } +repository = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[features] +aws = [ + "dep:aws-config", + "dep:aws-sdk-kms", + "dep:iceberg-property-macro", +] +default = [] + +[dependencies] +async-trait = { workspace = true } +aws-config = { workspace = true, optional = true } +aws-sdk-kms = { workspace = true, optional = true } +iceberg = { workspace = true } +iceberg-property-macro = { workspace = true, optional = true } + +[dev-dependencies] +aws-sdk-kms = { workspace = true, features = ["test-util"] } +aws-smithy-mocks = "0.3" +iceberg-catalog-rest = { workspace = true } +tokio = { workspace = true, features = ["macros"] } + +[lints] +workspace = true diff --git a/crates/kms/DEPENDENCIES.rust.tsv b/crates/kms/DEPENDENCIES.rust.tsv new file mode 100644 index 0000000000..1c4d66ae92 --- /dev/null +++ b/crates/kms/DEPENDENCIES.rust.tsv @@ -0,0 +1,305 @@ +crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC LGPL-2.1-or-later MIT Unicode-3.0 Unlicense Zlib +adler2@2.0.1 X X X +aead@0.5.2 X X +aes@0.8.4 X X +aes-gcm@0.10.3 X X +ahash@0.8.12 X X +aho-corasick@1.1.4 X X +alloc-no-stdlib@2.0.4 X +alloc-stdlib@0.2.4 X +android_system_properties@0.1.5 X X +anyhow@1.0.103 X X +apache-avro@0.21.0 X +array-init@2.1.0 X X +arrow-arith@59.3.0 X +arrow-array@59.3.0 X X +arrow-buffer@59.3.0 X +arrow-cast@59.3.0 X +arrow-data@59.3.0 X +arrow-ipc@59.3.0 X +arrow-ord@59.3.0 X +arrow-schema@59.3.0 X +arrow-select@59.3.0 X +arrow-string@59.3.0 X +as-any@0.3.2 X X +async-lock@3.4.2 X X +async-trait@0.1.89 X X +atoi@2.0.0 X +atomic-waker@1.1.2 X X +autocfg@1.5.1 X X +backon@1.6.0 X +base64@0.22.1 X X +base64@0.23.0 X X +bigdecimal@0.4.10 X X +bimap@0.6.3 X X +bitflags@2.13.0 X X +block-buffer@0.10.4 X X +bnum@0.12.1 X X +bon@3.9.3 X X +bon-macros@3.9.3 X X +brotli@8.0.4 X X +brotli-decompressor@5.0.3 X X +bumpalo@3.20.3 X X +bytemuck@1.25.0 X X X +bytemuck_derive@1.10.2 X X X +byteorder@1.5.0 X X +bytes@1.12.0 X +cc@1.2.66 X X +cfg-if@1.0.4 X X +chrono@0.4.45 X X +cipher@0.4.4 X X +concurrent-queue@2.5.0 X X +const-random@0.1.18 X X +const-random-macro@0.1.16 X X +core-foundation-sys@0.8.7 X X +cpufeatures@0.2.17 X X +crc32fast@1.5.0 X X +crossbeam-channel@0.5.16 X X +crossbeam-epoch@0.9.20 X X +crossbeam-utils@0.8.22 X X +crunchy@0.2.4 X +crypto-common@0.1.7 X X +ctr@0.9.2 X X +darling@0.20.11 X +darling@0.23.0 X +darling_core@0.20.11 X +darling_core@0.23.0 X +darling_macro@0.20.11 X +darling_macro@0.23.0 X +derive_builder@0.20.2 X X +derive_builder_core@0.20.2 X X +derive_builder_macro@0.20.2 X X +digest@0.10.7 X X +displaydoc@0.2.6 X X +dissimilar@1.0.11 X +either@1.16.0 X X +equivalent@1.0.2 X X +erased-serde@0.4.10 X X +event-listener@5.4.1 X X +event-listener-strategy@0.5.4 X X +expect-test@1.5.1 X X +fastnum@0.7.5 X X +fastrand@2.4.1 X X +find-msvc-tools@0.1.9 X X +flatbuffers@25.12.19 X +flate2@1.1.9 X X +fnv@1.0.7 X X +form_urlencoded@1.2.2 X X +futures@0.3.32 X X +futures-channel@0.3.32 X X +futures-core@0.3.32 X X +futures-executor@0.3.32 X X +futures-io@0.3.32 X X +futures-macro@0.3.32 X X +futures-sink@0.3.32 X X +futures-task@0.3.32 X X +futures-util@0.3.32 X X +generic-array@0.14.7 X +getrandom@0.2.17 X X +getrandom@0.3.4 X X +getrandom@0.4.3 X X +ghash@0.5.1 X X +gloo-timers@0.3.0 X X +h2@0.4.16 X +half@2.7.1 X X +hashbrown@0.17.1 X X +heck@0.5.0 X X +http@1.5.0 X X +http-body@1.0.1 X +http-body-util@0.1.3 X +httparse@1.10.1 X X +httpdate@1.0.3 X X +hyper@1.10.1 X +hyper-util@0.1.20 X +iana-time-zone@0.1.65 X X +iana-time-zone-haiku@0.1.2 X X +iceberg@0.10.1 X +iceberg-catalog-rest@0.10.1 X +iceberg-kms@0.10.1 X +iceberg-property-macro@0.10.1 X +iceberg_test_utils@0.10.1 X +icu_collections@2.2.0 X +icu_locale_core@2.2.0 X +icu_normalizer@2.2.0 X +icu_normalizer_data@2.2.0 X +icu_properties@2.2.0 X +icu_properties_data@2.2.0 X +icu_provider@2.2.0 X +ident_case@1.0.1 X X +idna@1.1.0 X X +idna_adapter@1.2.2 X X +indexmap@2.14.0 X X +inout@0.1.4 X X +inventory@0.3.24 X X +ipnet@2.12.0 X X +itertools@0.13.0 X X +itoa@1.0.18 X X +jobserver@0.1.35 X X +js-sys@0.3.103 X X +lazy_static@1.5.0 X X +lexical-core@1.0.6 X X +lexical-parse-float@1.0.6 X X +lexical-parse-integer@1.0.6 X X +lexical-util@1.0.7 X X +lexical-write-float@1.0.6 X X +lexical-write-integer@1.0.6 X X +libc@0.2.186 X X +libm@0.2.16 X +litemap@0.8.2 X +lock_api@0.4.14 X X +log@0.4.33 X X +lz4_flex@0.14.0 X +matchers@0.2.0 X +memchr@2.8.2 X X +miniz_oxide@0.8.9 X X X +mio@1.2.1 X +moka@0.12.15 X X +murmur3@0.5.2 X X +nu-ansi-term@0.50.3 X +num-bigint@0.4.8 X X +num-bigint@0.5.1 X X +num-complex@0.4.6 X X +num-integer@0.1.46 X X +num-traits@0.2.19 X X +once_cell@1.21.4 X X +opaque-debug@0.3.1 X X +ordered-float@4.6.0 X +parking@2.2.1 X X +parking_lot@0.12.5 X X +parking_lot_core@0.9.12 X X +parquet@59.3.0 X +percent-encoding@2.3.2 X X +pin-project-lite@0.2.17 X X +pkg-config@0.3.33 X X +polyval@0.6.2 X X +portable-atomic@1.13.1 X X +potential_utf@0.1.5 X +ppv-lite86@0.2.21 X X +prettyplease@0.2.37 X X +proc-macro2@1.0.106 X X +quad-rand@0.2.3 X +quote@1.0.46 X X +r-efi@5.3.0 X X X +r-efi@6.0.0 X X X +rand@0.9.5 X X +rand_chacha@0.9.0 X X +rand_core@0.6.4 X X +rand_core@0.9.5 X X +redox_syscall@0.5.18 X +regex@1.13.1 X X +regex-automata@0.4.16 X X +regex-lite@0.1.9 X X +regex-syntax@0.8.11 X X +reqwest@0.12.28 X X +ring@0.17.14 X X +roaring@0.11.4 X X +rustc_version@0.4.1 X X +rustversion@1.0.23 X X +ryu@1.0.23 X X +scopeguard@1.2.0 X X +semver@1.0.28 X X +seq-macro@0.3.6 X X +serde@1.0.228 X X +serde-big-array@0.5.1 X X +serde_bytes@0.11.19 X X +serde_core@1.0.228 X X +serde_derive@1.0.228 X X +serde_json@1.0.150 X X +serde_repr@0.1.20 X X +serde_urlencoded@0.7.1 X X +serde_with@3.22.0 X X +serde_with_macros@3.22.0 X X +sharded-slab@0.1.7 X +shlex@2.0.1 X X +simd-adler32@0.3.9 X +simdutf8@0.1.5 X X +slab@0.4.12 X +smallvec@1.15.2 X X +snap@1.1.1 X +socket2@0.6.4 X X +stable_deref_trait@1.2.1 X X +strsim@0.11.1 X +strum@0.27.2 X +strum_macros@0.27.2 X +subtle@2.6.1 X +syn@2.0.119 X X +sync_wrapper@1.0.2 X +synstructure@0.13.2 X +tagptr@0.2.0 X X +thiserror@2.0.18 X X +thiserror-impl@2.0.18 X X +thread_local@1.1.9 X X +tiny-keccak@2.0.2 X +tinystr@0.8.3 X +tokio@1.53.1 X +tokio-macros@2.7.0 X +tokio-util@0.7.18 X +tower@0.5.3 X +tower-http@0.6.11 X +tower-layer@0.3.3 X +tower-service@0.3.3 X +tracing@0.1.44 X +tracing-attributes@0.1.31 X +tracing-core@0.1.36 X +tracing-log@0.2.0 X +tracing-serde@0.2.0 X +tracing-subscriber@0.3.23 X +try-lock@0.2.5 X +twox-hash@2.1.2 X +typed-builder@0.20.1 X X +typed-builder-macro@0.20.1 X X +typeid@1.0.3 X X +typenum@1.20.1 X X +typetag@0.2.22 X X +typetag-impl@0.2.22 X X +unicode-ident@1.0.24 X X X +universal-hash@0.5.1 X X +untrusted@0.9.0 X +url@2.5.8 X X +utf8_iter@1.0.4 X X +uuid@1.26.0 X X +version_check@0.9.5 X X +want@0.3.1 X +wasi@0.11.1+wasi-snapshot-preview1 X X X +wasip2@1.0.4+wasi-0.2.12 X X X +wasm-bindgen@0.2.126 X X +wasm-bindgen-futures@0.4.76 X X +wasm-bindgen-macro@0.2.126 X X +wasm-bindgen-macro-support@0.2.126 X X +wasm-bindgen-shared@0.2.126 X X +wasm-streams@0.4.2 X X +web-sys@0.3.103 X X +windows-core@0.62.2 X X +windows-implement@0.60.2 X X +windows-interface@0.59.3 X X +windows-link@0.2.1 X X +windows-result@0.4.1 X X +windows-strings@0.5.1 X X +windows-sys@0.52.0 X X +windows-sys@0.61.2 X X +windows-targets@0.52.6 X X +windows_aarch64_gnullvm@0.52.6 X X +windows_aarch64_msvc@0.52.6 X X +windows_i686_gnu@0.52.6 X X +windows_i686_gnullvm@0.52.6 X X +windows_i686_msvc@0.52.6 X X +windows_x86_64_gnu@0.52.6 X X +windows_x86_64_gnullvm@0.52.6 X X +windows_x86_64_msvc@0.52.6 X X +wit-bindgen@0.57.1 X X X +writeable@0.6.3 X +yoke@0.8.3 X +yoke-derive@0.8.2 X +zerocopy@0.8.53 X X X +zerocopy-derive@0.8.53 X X X +zerofrom@0.1.8 X +zerofrom-derive@0.1.7 X +zeroize@1.9.0 X X +zerotrie@0.2.4 X +zerovec@0.11.6 X +zerovec-derive@0.11.3 X +zlib-rs@0.6.5 X +zmij@1.0.21 X +zstd@0.13.3 X +zstd-safe@7.2.4 X X +zstd-sys@2.0.16+zstd.1.5.7 X X diff --git a/crates/kms/LICENSE b/crates/kms/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/crates/kms/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/crates/kms/NOTICE b/crates/kms/NOTICE new file mode 120000 index 0000000000..fb376cfaa4 --- /dev/null +++ b/crates/kms/NOTICE @@ -0,0 +1 @@ +../../NOTICE \ No newline at end of file diff --git a/crates/kms/README.md b/crates/kms/README.md new file mode 100644 index 0000000000..fdf82d4142 --- /dev/null +++ b/crates/kms/README.md @@ -0,0 +1,47 @@ + + +# Apache Iceberg KMS + +This crate provides key management service integrations for Apache Iceberg +client-side encryption. Provider implementations are enabled through feature +flags. + +| Provider | Feature flag | +| -------------------------- | ------------ | +| AWS Key Management Service | `aws` | + +Add the crate with the feature for the provider you need: + +```toml +[dependencies] +iceberg-kms = { version = "x.y.z", features = ["aws"] } +``` + +`ResolvingKmsClientFactory` can select an enabled provider from the +`encryption.kms-type` catalog property. Built-in values follow Iceberg Java: +`aws`, `azure`, and `gcp`. Currently, only `aws` has a Rust implementation. + +The built-in mappings are fixed. Applications using a custom KMS provider can +supply their own implementation of `KmsClientFactory` directly to the catalog +builder. This replaces Java's reflection-based `encryption.kms-impl` extension +point. + +See the [API documentation](https://docs.rs/iceberg-kms/latest) for +provider-specific configuration and usage. diff --git a/crates/kms/public-api.txt b/crates/kms/public-api.txt new file mode 100644 index 0000000000..ce307e4177 --- /dev/null +++ b/crates/kms/public-api.txt @@ -0,0 +1,40 @@ +pub mod iceberg_kms +pub mod iceberg_kms::aws +pub struct iceberg_kms::aws::AwsKmsClientFactory +impl iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::new() -> Self +pub fn iceberg_kms::aws::AwsKmsClientFactory::with_sdk_config(self, sdk_config: aws_types::sdk_config::SdkConfig) -> Self +impl core::clone::Clone for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::clone(&self) -> iceberg_kms::aws::AwsKmsClientFactory +impl core::default::Default for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::default() -> iceberg_kms::aws::AwsKmsClientFactory +impl core::fmt::Debug for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::kms::factory::KmsClientFactory for iceberg_kms::aws::AwsKmsClientFactory +pub fn iceberg_kms::aws::AwsKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub const iceberg_kms::aws::AWS_ACCESS_KEY_ID: &str +pub const iceberg_kms::aws::AWS_PROFILE_NAME: &str +pub const iceberg_kms::aws::AWS_REGION_NAME: &str +pub const iceberg_kms::aws::AWS_SECRET_ACCESS_KEY: &str +pub const iceberg_kms::aws::AWS_SESSION_TOKEN: &str +pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC: &str +pub const iceberg_kms::aws::KMS_DATA_KEY_SPEC_DEFAULT: &str +pub const iceberg_kms::aws::KMS_ENCRYPTION_ALGORITHM_SPEC: &str +pub const iceberg_kms::aws::KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str +pub const iceberg_kms::aws::KMS_ENDPOINT: &str +pub struct iceberg_kms::ResolvingKmsClientFactory +impl iceberg_kms::ResolvingKmsClientFactory +pub fn iceberg_kms::ResolvingKmsClientFactory::new() -> Self +pub fn iceberg_kms::ResolvingKmsClientFactory::with_aws_factory(self, factory: iceberg_kms::aws::AwsKmsClientFactory) -> Self +impl core::clone::Clone for iceberg_kms::ResolvingKmsClientFactory +pub fn iceberg_kms::ResolvingKmsClientFactory::clone(&self) -> iceberg_kms::ResolvingKmsClientFactory +impl core::default::Default for iceberg_kms::ResolvingKmsClientFactory +pub fn iceberg_kms::ResolvingKmsClientFactory::default() -> Self +impl core::fmt::Debug for iceberg_kms::ResolvingKmsClientFactory +pub fn iceberg_kms::ResolvingKmsClientFactory::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl iceberg::encryption::kms::factory::KmsClientFactory for iceberg_kms::ResolvingKmsClientFactory +pub fn iceberg_kms::ResolvingKmsClientFactory::create_kms_client<'life0, 'life1, 'async_trait>(&'life0 self, properties: &'life1 std::collections::hash::map::HashMap) -> core::pin::Pin>> + core::marker::Send + 'async_trait)>> where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait +pub const iceberg_kms::ENCRYPTION_KMS_TYPE: &str +pub const iceberg_kms::ENCRYPTION_KMS_TYPE_AWS: &str +pub const iceberg_kms::ENCRYPTION_KMS_TYPE_AZURE: &str +pub const iceberg_kms::ENCRYPTION_KMS_TYPE_GCP: &str diff --git a/crates/kms/src/aws/client.rs b/crates/kms/src/aws/client.rs new file mode 100644 index 0000000000..3abd93cd3e --- /dev/null +++ b/crates/kms/src/aws/client.rs @@ -0,0 +1,390 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt; + +use async_trait::async_trait; +use aws_sdk_kms::Client; +use aws_sdk_kms::primitives::Blob; +use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; +use iceberg::encryption::{GeneratedKey, KeyManagementClient, SensitiveBytes}; +use iceberg::{Error, ErrorKind, Result}; + +pub(crate) struct AwsKeyManagementClient { + client: Client, + encryption_algorithm: EncryptionAlgorithmSpec, + data_key_spec: DataKeySpec, +} + +impl AwsKeyManagementClient { + pub(crate) fn new( + client: Client, + encryption_algorithm: EncryptionAlgorithmSpec, + data_key_spec: DataKeySpec, + ) -> Self { + Self { + client, + encryption_algorithm, + data_key_spec, + } + } + + fn missing_response_field(operation: &str, field: &str) -> Error { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS {operation} response did not contain {field}"), + ) + } +} + +impl fmt::Debug for AwsKeyManagementClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AwsKeyManagementClient") + .field("encryption_algorithm", &self.encryption_algorithm) + .field("data_key_spec", &self.data_key_spec) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl KeyManagementClient for AwsKeyManagementClient { + async fn wrap_key(&self, key: &[u8], wrapping_key_id: &str) -> Result> { + let response = self + .client + .encrypt() + .key_id(wrapping_key_id) + .encryption_algorithm(self.encryption_algorithm.clone()) + .plaintext(Blob::new(key)) + .send() + .await + .map_err(|source| { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS Encrypt failed for key '{wrapping_key_id}'"), + ) + .with_source(source) + })?; + + response + .ciphertext_blob + .map(Blob::into_inner) + .ok_or_else(|| Self::missing_response_field("Encrypt", "ciphertext_blob")) + } + + async fn unwrap_key( + &self, + wrapped_key: &[u8], + wrapping_key_id: &str, + ) -> Result { + let response = self + .client + .decrypt() + .key_id(wrapping_key_id) + .encryption_algorithm(self.encryption_algorithm.clone()) + .ciphertext_blob(Blob::new(wrapped_key)) + .send() + .await + .map_err(|source| { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS Decrypt failed for key '{wrapping_key_id}'"), + ) + .with_source(source) + })?; + + response + .plaintext + .map(|blob| SensitiveBytes::new(blob.into_inner())) + .ok_or_else(|| Self::missing_response_field("Decrypt", "plaintext")) + } + + fn supports_key_generation(&self) -> bool { + self.encryption_algorithm == EncryptionAlgorithmSpec::SymmetricDefault + } + + async fn generate_key(&self, wrapping_key_id: &str) -> Result { + let response = self + .client + .generate_data_key() + .key_id(wrapping_key_id) + .key_spec(self.data_key_spec.clone()) + .send() + .await + .map_err(|source| { + Error::new( + ErrorKind::Unexpected, + format!("AWS KMS GenerateDataKey failed for key '{wrapping_key_id}'"), + ) + .with_source(source) + })?; + + let plaintext = response + .plaintext + .map(|blob| SensitiveBytes::new(blob.into_inner())) + .ok_or_else(|| Self::missing_response_field("GenerateDataKey", "plaintext"))?; + let wrapped_key = response + .ciphertext_blob + .map(Blob::into_inner) + .ok_or_else(|| Self::missing_response_field("GenerateDataKey", "ciphertext_blob"))?; + + Ok(GeneratedKey::new(plaintext, wrapped_key)) + } +} + +#[cfg(test)] +mod tests { + use std::error::Error as StdError; + use std::sync::{Arc, Mutex}; + + use aws_sdk_kms::Client; + use aws_sdk_kms::operation::decrypt::DecryptOutput; + use aws_sdk_kms::operation::encrypt::{EncryptError, EncryptOutput}; + use aws_sdk_kms::operation::generate_data_key::GenerateDataKeyOutput; + use aws_sdk_kms::types::error::NotFoundException; + use aws_smithy_mocks::{RuleMode, mock, mock_client}; + use iceberg::encryption::{EncryptionManager, StandardKeyMetadata}; + + use super::*; + + const KEY_ID: &str = "arn:aws:kms:eu-west-2:123456789012:key/test-key"; + + #[tokio::test] + async fn test_wrap_key() { + let plaintext = b"0123456789abcdef"; + let ciphertext = b"wrapped-key"; + let rule = mock!(Client::encrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::SymmetricDefault) + && request.plaintext().map(AsRef::as_ref) == Some(plaintext.as_slice()) + }) + .then_output(move || { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(ciphertext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + assert_eq!(kms.wrap_key(plaintext, KEY_ID).await.unwrap(), ciphertext); + assert_eq!(rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_unwrap_key() { + let plaintext = b"0123456789abcdef"; + let ciphertext = b"wrapped-key"; + let rule = mock!(Client::decrypt) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) + && request.encryption_algorithm() + == Some(&EncryptionAlgorithmSpec::SymmetricDefault) + && request.ciphertext_blob().map(AsRef::as_ref) == Some(ciphertext.as_slice()) + }) + .then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + assert_eq!( + kms.unwrap_key(ciphertext, KEY_ID).await.unwrap().as_bytes(), + plaintext + ); + assert_eq!(rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_generate_key() { + let plaintext = b"0123456789abcdef0123456789abcdef"; + let ciphertext = b"generated-wrapped-key"; + let rule = mock!(Client::generate_data_key) + .match_requests(|request| { + request.key_id() == Some(KEY_ID) && request.key_spec() == Some(&DataKeySpec::Aes256) + }) + .then_output(move || { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new(plaintext)) + .ciphertext_blob(Blob::new(ciphertext)) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + let generated = kms.generate_key(KEY_ID).await.unwrap(); + assert_eq!(generated.key().as_bytes(), plaintext); + assert_eq!(generated.wrapped_key(), ciphertext); + assert_eq!(rule.num_calls(), 1); + } + + #[test] + fn test_key_generation_support_depends_on_encryption_algorithm() { + let client = mock_client!(aws_sdk_kms, &[]); + + assert!( + AwsKeyManagementClient::new( + client.clone(), + EncryptionAlgorithmSpec::SymmetricDefault, + DataKeySpec::Aes256, + ) + .supports_key_generation() + ); + assert!( + !AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::RsaesOaepSha256, + DataKeySpec::Aes256, + ) + .supports_key_generation() + ); + } + + #[tokio::test] + async fn test_asymmetric_manager_roundtrip_uses_encrypt_instead_of_generate_data_key() { + let plaintext_kek = Arc::new(Mutex::new(Vec::new())); + let plaintext_kek_from_encrypt = Arc::clone(&plaintext_kek); + let encrypt_rule = mock!(Client::encrypt) + .match_requests(move |request| { + if request.key_id() != Some(KEY_ID) + || request.encryption_algorithm() + != Some(&EncryptionAlgorithmSpec::RsaesOaepSha256) + { + return false; + } + + *plaintext_kek_from_encrypt.lock().unwrap() = request + .plaintext() + .expect("EncryptionManager must provide a plaintext KEK") + .as_ref() + .to_vec(); + true + }) + .then_output(|| { + EncryptOutput::builder() + .ciphertext_blob(Blob::new(b"wrapped-asymmetric-kek")) + .build() + }); + let plaintext_kek_for_decrypt = Arc::clone(&plaintext_kek); + let decrypt_rule = mock!(Client::decrypt).then_output(move || { + DecryptOutput::builder() + .plaintext(Blob::new(plaintext_kek_for_decrypt.lock().unwrap().clone())) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&encrypt_rule, &decrypt_rule]); + let kms: Arc = Arc::new(AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::RsaesOaepSha256, + DataKeySpec::Aes256, + )); + let manager = EncryptionManager::builder() + .kms_client(Arc::clone(&kms)) + .table_key_id(KEY_ID) + .build(); + let key_metadata = StandardKeyMetadata::try_new(b"0123456789abcdef") + .unwrap() + .with_aad_prefix(b"test-aad-prefix!"); + + let encrypted_key_id = manager + .encrypt_manifest_list_key_metadata(&key_metadata) + .await + .unwrap(); + let encryption_keys = manager.with_encryption_keys(Clone::clone); + let reader = EncryptionManager::builder() + .kms_client(kms) + .table_key_id(KEY_ID) + .encryption_keys(encryption_keys) + .build(); + let decrypted = reader + .decrypt_manifest_list_key_metadata(&encrypted_key_id) + .await + .unwrap(); + + assert_eq!(decrypted, key_metadata); + assert_eq!(encrypt_rule.num_calls(), 1); + assert_eq!(decrypt_rule.num_calls(), 1); + } + + #[tokio::test] + async fn test_aws_service_error_is_preserved_as_source() { + let rule = mock!(Client::encrypt).then_error(|| { + EncryptError::NotFoundException( + NotFoundException::builder() + .message("test key not found") + .build(), + ) + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + let error = kms.wrap_key(b"key", KEY_ID).await.unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.source().is_some()); + } + + #[tokio::test] + async fn test_missing_response_fields() { + let encrypt_rule = mock!(Client::encrypt).then_output(|| EncryptOutput::builder().build()); + let decrypt_rule = mock!(Client::decrypt).then_output(|| DecryptOutput::builder().build()); + let generate_rule = mock!(Client::generate_data_key) + .then_output(|| GenerateDataKeyOutput::builder().build()); + let client = mock_client!(aws_sdk_kms, RuleMode::Sequential, &[ + &encrypt_rule, + &decrypt_rule, + &generate_rule + ]); + let kms = test_client(client); + + let error = kms.wrap_key(b"key", KEY_ID).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("ciphertext_blob")); + + let error = kms.unwrap_key(b"wrapped", KEY_ID).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("plaintext")); + + let error = kms.generate_key(KEY_ID).await.err().unwrap(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("plaintext")); + } + + #[tokio::test] + async fn test_generate_key_requires_wrapped_key() { + let rule = mock!(Client::generate_data_key).then_output(|| { + GenerateDataKeyOutput::builder() + .plaintext(Blob::new([0; 32])) + .build() + }); + let client = mock_client!(aws_sdk_kms, &[&rule]); + let kms = test_client(client); + + let error = kms.generate_key(KEY_ID).await.err().unwrap(); + assert_eq!(error.kind(), ErrorKind::Unexpected); + assert!(error.message().contains("ciphertext_blob")); + } + + fn test_client(client: Client) -> AwsKeyManagementClient { + AwsKeyManagementClient::new( + client, + EncryptionAlgorithmSpec::SymmetricDefault, + DataKeySpec::Aes256, + ) + } +} diff --git a/crates/kms/src/aws/config.rs b/crates/kms/src/aws/config.rs new file mode 100644 index 0000000000..925c1aa4f5 --- /dev/null +++ b/crates/kms/src/aws/config.rs @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use aws_sdk_kms::types::{DataKeySpec, EncryptionAlgorithmSpec}; +use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; + +/// Catalog property selecting an AWS profile. +pub const AWS_PROFILE_NAME: &str = "profile_name"; +/// Catalog property selecting the AWS region. +pub const AWS_REGION_NAME: &str = "region_name"; +/// Catalog property containing a static AWS access key ID. +pub const AWS_ACCESS_KEY_ID: &str = "aws_access_key_id"; +/// Catalog property containing a static AWS secret access key. +pub const AWS_SECRET_ACCESS_KEY: &str = "aws_secret_access_key"; +/// Catalog property containing a static AWS session token. +pub const AWS_SESSION_TOKEN: &str = "aws_session_token"; + +/// Catalog property overriding the AWS KMS service endpoint. +pub const KMS_ENDPOINT: &str = "kms.endpoint"; +/// Catalog property selecting the AWS KMS encryption algorithm. +pub const KMS_ENCRYPTION_ALGORITHM_SPEC: &str = "kms.encryption-algorithm-spec"; +/// Default AWS KMS encryption algorithm, matching Iceberg Java. +pub const KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT: &str = "SYMMETRIC_DEFAULT"; +/// Catalog property selecting the size of keys generated by AWS KMS. +pub const KMS_DATA_KEY_SPEC: &str = "kms.data-key-spec"; +/// Default AWS KMS generated key size, matching Iceberg Java. +pub const KMS_DATA_KEY_SPEC_DEFAULT: &str = "AES_256"; + +#[derive(Clone, Properties)] +pub(crate) struct AwsKmsConfig { + #[property(key = KMS_ENDPOINT, default = None)] + pub(crate) endpoint: Option, + #[property( + key = KMS_ENCRYPTION_ALGORITHM_SPEC, + default = KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT, + parse_with = parse_encryption_algorithm + )] + pub(crate) encryption_algorithm: EncryptionAlgorithmSpec, + #[property( + key = KMS_DATA_KEY_SPEC, + default = KMS_DATA_KEY_SPEC_DEFAULT, + parse_with = parse_data_key_spec + )] + pub(crate) data_key_spec: DataKeySpec, +} + +fn parse_encryption_algorithm(value: &str) -> Result { + EncryptionAlgorithmSpec::try_parse(value).map_err(|source| { + Error::new( + ErrorKind::DataInvalid, + format!("Unsupported AWS KMS encryption algorithm: {value}"), + ) + .with_source(source) + }) +} + +fn parse_data_key_spec(value: &str) -> Result { + DataKeySpec::try_parse(value).map_err(|source| { + Error::new( + ErrorKind::DataInvalid, + format!("Unsupported AWS KMS data key spec: {value}"), + ) + .with_source(source) + }) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + #[test] + fn test_defaults_match_iceberg_java() { + let config = AwsKmsConfig::from_properties(&HashMap::new()).unwrap(); + + assert_eq!( + config.encryption_algorithm, + EncryptionAlgorithmSpec::SymmetricDefault + ); + assert_eq!(config.data_key_spec, DataKeySpec::Aes256); + assert_eq!(config.endpoint, None); + } + + #[test] + fn test_parse_overrides() { + let properties = HashMap::from([ + ( + KMS_ENDPOINT.to_string(), + "http://localhost:4566".to_string(), + ), + ( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "RSAES_OAEP_SHA_256".to_string(), + ), + (KMS_DATA_KEY_SPEC.to_string(), "AES_128".to_string()), + ]); + + let config = AwsKmsConfig::from_properties(&properties).unwrap(); + assert_eq!(config.endpoint.as_deref(), Some("http://localhost:4566")); + assert_eq!( + config.encryption_algorithm, + EncryptionAlgorithmSpec::RsaesOaepSha256 + ); + assert_eq!(config.data_key_spec, DataKeySpec::Aes128); + } + + #[test] + fn test_reject_unknown_values() { + let bad_algorithm = HashMap::from([( + KMS_ENCRYPTION_ALGORITHM_SPEC.to_string(), + "unknown".to_string(), + )]); + let Err(error) = AwsKmsConfig::from_properties(&bad_algorithm) else { + panic!("unknown encryption algorithms must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + + let bad_key_spec = HashMap::from([(KMS_DATA_KEY_SPEC.to_string(), "AES_192".to_string())]); + let Err(error) = AwsKmsConfig::from_properties(&bad_key_spec) else { + panic!("unknown data-key specifications must be rejected"); + }; + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } +} diff --git a/crates/kms/src/aws/factory.rs b/crates/kms/src/aws/factory.rs new file mode 100644 index 0000000000..9bbd338d18 --- /dev/null +++ b/crates/kms/src/aws/factory.rs @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use async_trait::async_trait; +use aws_config::{BehaviorVersion, SdkConfig}; +use aws_sdk_kms::Client as KmsClient; +use aws_sdk_kms::config::{Builder as KmsConfigBuilder, Credentials, Region}; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::{Error, ErrorKind, Result}; + +use super::client::AwsKeyManagementClient; +use super::config::{ + AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + AwsKmsConfig, +}; + +/// Factory for AWS KMS-backed Iceberg key management clients. +/// +/// By default, AWS credentials and region are resolved through the AWS SDK's +/// standard provider chains. Catalog properties can override the profile, +/// region, or static credentials. Applications with custom credential or HTTP +/// providers can supply a complete [`SdkConfig`] through +/// [`with_sdk_config`](Self::with_sdk_config). +#[derive(Clone, Default)] +pub struct AwsKmsClientFactory { + sdk_config: Option, +} + +impl AwsKmsClientFactory { + /// Create a factory using catalog properties and the AWS SDK default chains. + pub fn new() -> Self { + Self::default() + } + + /// Use an application-provided AWS SDK configuration. + /// + /// When set, SDK-wide credential, region, retry, and HTTP configuration + /// properties are not rebuilt from catalog properties. KMS-specific + /// properties such as `kms.endpoint` continue to be applied. + pub fn with_sdk_config(mut self, sdk_config: SdkConfig) -> Self { + self.sdk_config = Some(sdk_config); + self + } +} + +impl fmt::Debug for AwsKmsClientFactory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AwsKmsClientFactory") + .field("has_sdk_config", &self.sdk_config.is_some()) + .finish() + } +} + +#[async_trait] +impl KmsClientFactory for AwsKmsClientFactory { + async fn create_kms_client( + &self, + properties: &HashMap, + ) -> Result> { + let kms_config = AwsKmsConfig::from_properties(properties)?; + let sdk_config = match &self.sdk_config { + Some(config) => config.clone(), + None => create_sdk_config(properties).await?, + }; + + let mut builder = KmsConfigBuilder::from(&sdk_config); + if sdk_config.behavior_version().is_none() { + builder = builder.behavior_version(BehaviorVersion::latest()); + } + if let Some(endpoint) = &kms_config.endpoint { + builder = builder.endpoint_url(endpoint); + } + + Ok(Arc::new(AwsKeyManagementClient::new( + KmsClient::from_conf(builder.build()), + kms_config.encryption_algorithm, + kms_config.data_key_spec, + ))) + } +} + +async fn create_sdk_config(properties: &HashMap) -> Result { + let mut loader = aws_config::defaults(BehaviorVersion::latest()); + + match ( + properties.get(AWS_ACCESS_KEY_ID), + properties.get(AWS_SECRET_ACCESS_KEY), + ) { + (Some(access_key), Some(secret_key)) => { + let credentials = Credentials::new( + access_key, + secret_key, + properties.get(AWS_SESSION_TOKEN).cloned(), + None, + "catalog-properties", + ); + loader = loader.credentials_provider(credentials); + } + (None, None) => { + if properties.contains_key(AWS_SESSION_TOKEN) { + return Err(incomplete_credentials_error()); + } + } + _ => return Err(incomplete_credentials_error()), + } + + if let Some(profile) = properties.get(AWS_PROFILE_NAME) { + loader = loader.profile_name(profile); + } + if let Some(region) = properties.get(AWS_REGION_NAME) { + loader = loader.region(Region::new(region.clone())); + } + + Ok(loader.load().await) +} + +fn incomplete_credentials_error() -> Error { + Error::new( + ErrorKind::DataInvalid, + format!( + "AWS static credentials require both '{AWS_ACCESS_KEY_ID}' and '{AWS_SECRET_ACCESS_KEY}'" + ), + ) +} + +#[cfg(test)] +mod tests { + use aws_config::SdkConfig; + use aws_sdk_kms::config::ProvideCredentials; + use iceberg::encryption::kms::KmsClientFactory; + + use super::*; + use crate::aws::KMS_ENDPOINT; + + #[tokio::test] + async fn test_create_client_with_injected_sdk_config() { + let factory = AwsKmsClientFactory::new().with_sdk_config(test_sdk_config()); + let properties = HashMap::from([( + KMS_ENDPOINT.to_string(), + "http://localhost:4566".to_string(), + )]); + + let client = factory.create_kms_client(&properties).await.unwrap(); + assert!(client.supports_key_generation()); + } + + #[tokio::test] + async fn test_create_client_adds_missing_behavior_version() { + let factory = AwsKmsClientFactory::new().with_sdk_config(SdkConfig::builder().build()); + + let client = factory.create_kms_client(&HashMap::new()).await.unwrap(); + + assert!(client.supports_key_generation()); + } + + #[tokio::test] + async fn test_reject_incomplete_static_credentials() { + let only_access_key = + HashMap::from([(AWS_ACCESS_KEY_ID.to_string(), "access".to_string())]); + let error = create_sdk_config(&only_access_key).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + + let only_session_token = + HashMap::from([(AWS_SESSION_TOKEN.to_string(), "token".to_string())]); + let error = create_sdk_config(&only_session_token).await.unwrap_err(); + assert_eq!(error.kind(), ErrorKind::DataInvalid); + } + + #[tokio::test] + async fn test_static_credentials_and_region_override_default_chains() { + let properties = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "access".to_string()), + (AWS_SECRET_ACCESS_KEY.to_string(), "secret".to_string()), + (AWS_SESSION_TOKEN.to_string(), "token".to_string()), + (AWS_REGION_NAME.to_string(), "eu-west-2".to_string()), + ]); + + let config = create_sdk_config(&properties).await.unwrap(); + let credentials = config + .credentials_provider() + .expect("static credentials must install a credentials provider") + .provide_credentials() + .await + .unwrap(); + + assert_eq!(config.region().map(Region::as_ref), Some("eu-west-2")); + assert_eq!(credentials.access_key_id(), "access"); + assert_eq!(credentials.secret_access_key(), "secret"); + assert_eq!(credentials.session_token(), Some("token")); + } + + fn test_sdk_config() -> SdkConfig { + SdkConfig::builder() + .behavior_version(BehaviorVersion::latest()) + .build() + } +} diff --git a/crates/kms/src/aws/mod.rs b/crates/kms/src/aws/mod.rs new file mode 100644 index 0000000000..9782e7ac2b --- /dev/null +++ b/crates/kms/src/aws/mod.rs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! AWS Key Management Service integration. +//! +//! This module provides [`AwsKmsClientFactory`](crate::aws::AwsKmsClientFactory), +//! which creates AWS-backed implementations of Iceberg's +//! [`KeyManagementClient`](iceberg::encryption::KeyManagementClient). The factory can +//! be supplied to any Iceberg catalog builder through +//! [`CatalogBuilder::with_kms_client_factory`](iceberg::CatalogBuilder::with_kms_client_factory). +//! +//! The KMS key referenced by a table's `encryption.key-id` property requires +//! `kms:Decrypt` for reads. Writes with `SYMMETRIC_DEFAULT` use +//! `kms:GenerateDataKey`. Writes with an asymmetric encryption algorithm +//! generate the key-encryption key locally and require `kms:Encrypt` to wrap it. +//! +//! # Configuration +//! +//! The factory uses the AWS SDK's default credential and region provider chains. +//! The `profile_name`, `region_name`, `aws_access_key_id`, +//! `aws_secret_access_key`, and `aws_session_token` catalog properties can +//! override those defaults, following the conventions used by Iceberg Rust's +//! other AWS integrations. +//! +//! Catalog properties are the properties supplied by the application when the +//! catalog is constructed. Configuration returned later by a REST catalog +//! server is not currently available when the KMS client is created. +//! +//! The following KMS properties match Iceberg Java: +//! +//! - `kms.endpoint` overrides the KMS service endpoint. +//! - `kms.encryption-algorithm-spec` selects the KMS encryption algorithm and +//! defaults to `SYMMETRIC_DEFAULT`. +//! - `kms.data-key-spec` selects the size of KMS-generated key-encryption keys +//! and defaults to `AES_256`. +//! +//! `kms.data-key-spec` is independent from the Iceberg table property +//! `encryption.data-key-length`, which controls locally generated file data keys. +//! +//! # Example +//! +//! ```rust,no_run +//! use std::collections::HashMap; +//! use std::sync::Arc; +//! +//! use iceberg::CatalogBuilder; +//! use iceberg_catalog_rest::RestCatalogBuilder; +//! use iceberg_kms::aws::AwsKmsClientFactory; +//! +//! # async fn example() -> iceberg::Result<()> { +//! let catalog = RestCatalogBuilder::default() +//! .with_kms_client_factory(Arc::new(AwsKmsClientFactory::new())) +//! .load("rest", HashMap::new()) +//! .await?; +//! # let _ = catalog; +//! # Ok(()) +//! # } +//! ``` + +mod client; +mod config; +mod factory; + +pub use config::{ + AWS_ACCESS_KEY_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, + KMS_DATA_KEY_SPEC, KMS_DATA_KEY_SPEC_DEFAULT, KMS_ENCRYPTION_ALGORITHM_SPEC, + KMS_ENCRYPTION_ALGORITHM_SPEC_DEFAULT, KMS_ENDPOINT, +}; +pub use factory::AwsKmsClientFactory; diff --git a/crates/kms/src/lib.rs b/crates/kms/src/lib.rs new file mode 100644 index 0000000000..beb9119d68 --- /dev/null +++ b/crates/kms/src/lib.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key management service integrations for Apache Iceberg client-side encryption. +//! +//! Provider implementations are enabled through crate features. Enable `aws` +//! to use the `aws` module. [`ResolvingKmsClientFactory`] selects an enabled +//! provider from the `encryption.kms-type` catalog property. + +#![deny(missing_docs)] + +/// AWS Key Management Service integration. +#[cfg(feature = "aws")] +pub mod aws; + +mod resolving; + +pub use resolving::{ + ENCRYPTION_KMS_TYPE, ENCRYPTION_KMS_TYPE_AWS, ENCRYPTION_KMS_TYPE_AZURE, + ENCRYPTION_KMS_TYPE_GCP, ResolvingKmsClientFactory, +}; diff --git a/crates/kms/src/resolving.rs b/crates/kms/src/resolving.rs new file mode 100644 index 0000000000..6a075aaa84 --- /dev/null +++ b/crates/kms/src/resolving.rs @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use async_trait::async_trait; +use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; +use iceberg::{Error, ErrorKind, Result}; + +#[cfg(feature = "aws")] +use crate::aws::AwsKmsClientFactory; + +/// Catalog property selecting the key management service implementation. +pub const ENCRYPTION_KMS_TYPE: &str = "encryption.kms-type"; +/// [`ENCRYPTION_KMS_TYPE`] value selecting AWS Key Management Service. +pub const ENCRYPTION_KMS_TYPE_AWS: &str = "aws"; +/// [`ENCRYPTION_KMS_TYPE`] value selecting Azure Key Vault. +pub const ENCRYPTION_KMS_TYPE_AZURE: &str = "azure"; +/// [`ENCRYPTION_KMS_TYPE`] value selecting Google Cloud Key Management Service. +pub const ENCRYPTION_KMS_TYPE_GCP: &str = "gcp"; + +const ENCRYPTION_KMS_IMPL: &str = "encryption.kms-impl"; + +/// A KMS client factory that resolves an implementation from catalog properties. +/// +/// The factory reads [`ENCRYPTION_KMS_TYPE`] and delegates client creation to the +/// corresponding provider factory. These built-in mappings are fixed: custom +/// implementations should instead be supplied directly through +/// [`CatalogBuilder::with_kms_client_factory`](iceberg::CatalogBuilder::with_kms_client_factory). +/// +/// Provider implementations must be enabled through their corresponding crate +/// features. The `aws` feature currently provides the `aws` KMS type. +/// +/// Unlike Iceberg Java, this factory does not load class names from +/// `encryption.kms-impl`. Rust applications use the `KmsClientFactory` trait as +/// the custom implementation extension point. Configuring `encryption.kms-impl` +/// is rejected with guidance to supply a custom factory directly. +/// +/// Catalog properties returned later by a REST catalog server are not currently +/// available when the KMS client is created. Until REST initialization is +/// updated, `encryption.kms-type` must be supplied by the application. +/// +/// # Example +/// +/// This example requires the `aws` feature. +/// +/// ```rust,no_run +/// use std::collections::HashMap; +/// use std::sync::Arc; +/// +/// use iceberg::CatalogBuilder; +/// use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalogBuilder}; +/// use iceberg_kms::{ENCRYPTION_KMS_TYPE, ResolvingKmsClientFactory}; +/// +/// # async fn example() -> iceberg::Result<()> { +/// let properties = HashMap::from([ +/// ( +/// REST_CATALOG_PROP_URI.to_string(), +/// "https://catalog.example.com".to_string(), +/// ), +/// (ENCRYPTION_KMS_TYPE.to_string(), "aws".to_string()), +/// ]); +/// let catalog = RestCatalogBuilder::default() +/// .with_kms_client_factory(Arc::new(ResolvingKmsClientFactory::new())) +/// .load("rest", properties) +/// .await?; +/// # let _ = catalog; +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Debug)] +pub struct ResolvingKmsClientFactory { + #[cfg(feature = "aws")] + aws_factory: AwsKmsClientFactory, +} + +impl Default for ResolvingKmsClientFactory { + fn default() -> Self { + Self::new() + } +} + +impl ResolvingKmsClientFactory { + /// Create a factory containing all enabled built-in KMS implementations. + pub fn new() -> Self { + Self { + #[cfg(feature = "aws")] + aws_factory: AwsKmsClientFactory::new(), + } + } + + /// Use a configured AWS KMS client factory for the `aws` KMS type. + #[cfg(feature = "aws")] + pub fn with_aws_factory(mut self, factory: AwsKmsClientFactory) -> Self { + self.aws_factory = factory; + self + } +} + +#[async_trait] +impl KmsClientFactory for ResolvingKmsClientFactory { + async fn create_kms_client( + &self, + properties: &HashMap, + ) -> Result> { + let kms_type = properties.get(ENCRYPTION_KMS_TYPE); + let kms_impl = properties.get(ENCRYPTION_KMS_IMPL); + + if let Some(kms_impl) = kms_impl { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + format!( + "KMS implementation class '{kms_impl}' is not supported; provide a custom KmsClientFactory instead" + ), + )); + } + + let kms_type = kms_type + .filter(|kms_type| !kms_type.is_empty()) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Missing required catalog property '{ENCRYPTION_KMS_TYPE}'"), + ) + })?; + + match kms_type.to_ascii_lowercase().as_str() { + ENCRYPTION_KMS_TYPE_AWS => { + #[cfg(feature = "aws")] + { + self.aws_factory.create_kms_client(properties).await + } + #[cfg(not(feature = "aws"))] + { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "KMS type 'aws' requires the 'aws' feature of iceberg-kms", + )) + } + } + ENCRYPTION_KMS_TYPE_AZURE | ENCRYPTION_KMS_TYPE_GCP => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("KMS type '{kms_type}' is not implemented by this version of iceberg-kms"), + )), + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported KMS type: {kms_type}"), + )), + } + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "aws")] + use aws_config::{BehaviorVersion, SdkConfig}; + + use super::*; + + #[cfg(feature = "aws")] + #[tokio::test] + async fn test_resolves_aws_case_insensitively() { + let factory = ResolvingKmsClientFactory::new().with_aws_factory( + AwsKmsClientFactory::new().with_sdk_config( + SdkConfig::builder() + .behavior_version(BehaviorVersion::latest()) + .build(), + ), + ); + let properties = HashMap::from([(ENCRYPTION_KMS_TYPE.to_string(), "AWS".to_string())]); + + let client = factory.create_kms_client(&properties).await.unwrap(); + + assert!(client.supports_key_generation()); + } + + #[tokio::test] + async fn test_rejects_missing_kms_type() { + let error = ResolvingKmsClientFactory::new() + .create_kms_client(&HashMap::new()) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::DataInvalid); + assert!(error.to_string().contains(ENCRYPTION_KMS_TYPE)); + } + + #[tokio::test] + async fn test_rejects_unsupported_kms_type() { + let properties = + HashMap::from([(ENCRYPTION_KMS_TYPE.to_string(), "unsupported".to_string())]); + let error = ResolvingKmsClientFactory::new() + .create_kms_client(&properties) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::FeatureUnsupported); + assert!(error.to_string().contains("unsupported")); + } + + #[tokio::test] + async fn test_reports_unimplemented_built_in_kms_types() { + for kms_type in [ENCRYPTION_KMS_TYPE_AZURE, ENCRYPTION_KMS_TYPE_GCP] { + let properties = + HashMap::from([(ENCRYPTION_KMS_TYPE.to_string(), kms_type.to_string())]); + let error = ResolvingKmsClientFactory::new() + .create_kms_client(&properties) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::FeatureUnsupported); + assert!(error.to_string().contains(kms_type)); + } + } + + #[tokio::test] + async fn test_rejects_java_kms_impl() { + let properties = HashMap::from([( + ENCRYPTION_KMS_IMPL.to_string(), + "com.example.CustomKmsClient".to_string(), + )]); + let error = ResolvingKmsClientFactory::new() + .create_kms_client(&properties) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::FeatureUnsupported); + assert!(error.to_string().contains("custom KmsClientFactory")); + } + + #[cfg(not(feature = "aws"))] + #[tokio::test] + async fn test_reports_disabled_built_in_kms_type() { + let properties = HashMap::from([( + ENCRYPTION_KMS_TYPE.to_string(), + ENCRYPTION_KMS_TYPE_AWS.to_string(), + )]); + let error = ResolvingKmsClientFactory::new() + .create_kms_client(&properties) + .await + .unwrap_err(); + + assert_eq!(error.kind(), ErrorKind::FeatureUnsupported); + assert!(error.to_string().contains(ENCRYPTION_KMS_TYPE_AWS)); + } +}