From ddb1cfd66482d003b488e79f3912705f213214e8 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Mon, 14 Sep 2026 14:02:23 -0400 Subject: [PATCH 1/5] update cargo to updated iceberg-rust --- Cargo.lock | 337 ++++++++++++++++++++------- Cargo.toml | 6 +- deny.toml | 8 +- src/storage-types/src/connections.rs | 14 +- 4 files changed, 265 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 834fe8cec7d75..27f43e01a9897 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -873,6 +873,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "asynchronous-codec" version = "0.6.0" @@ -1713,7 +1723,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -1803,7 +1813,7 @@ dependencies = [ "bitflags 2.11.0", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.10.5", "proc-macro2", "quote", "regex", @@ -2229,7 +2239,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -2688,15 +2698,6 @@ dependencies = [ "spin 0.10.1", ] -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -3200,6 +3201,37 @@ dependencies = [ "libc", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "der" version = "0.7.10" @@ -4278,9 +4310,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -4908,7 +4942,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.5", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -4933,7 +4967,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=985929c6c5cc7083bac30c881e7ec43ec4d57b64#985929c6c5cc7083bac30c881e7ec43ec4d57b64" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=eafda25053000154846871b36ed204e1bac03fb7#eafda25053000154846871b36ed204e1bac03fb7" dependencies = [ "aes-gcm", "anyhow", @@ -4990,7 +5024,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.10.1" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=985929c6c5cc7083bac30c881e7ec43ec4d57b64#985929c6c5cc7083bac30c881e7ec43ec4d57b64" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=eafda25053000154846871b36ed204e1bac03fb7#eafda25053000154846871b36ed204e1bac03fb7" dependencies = [ "async-trait", "chrono", @@ -5009,7 +5043,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=985929c6c5cc7083bac30c881e7ec43ec4d57b64#985929c6c5cc7083bac30c881e7ec43ec4d57b64" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=eafda25053000154846871b36ed204e1bac03fb7#eafda25053000154846871b36ed204e1bac03fb7" dependencies = [ "anyhow", "async-trait", @@ -5019,6 +5053,7 @@ dependencies = [ "iceberg", "opendal", "reqsign-aws-v4", + "reqsign-azure-storage", "reqsign-core", "reqsign-google", "serde", @@ -5132,7 +5167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.15.3", "serde", "serde_core", ] @@ -5241,7 +5276,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.1", + "windows-sys 0.59.0", ] [[package]] @@ -5318,10 +5353,12 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -5330,15 +5367,26 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.61.1", + "windows-link 0.2.1", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.119", @@ -5373,7 +5421,7 @@ dependencies = [ "simd_cesu8", "thiserror 2.0.18", "walkdir", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -5832,7 +5880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -5974,9 +6022,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "loom" @@ -6000,6 +6048,12 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "lsp-types" version = "0.94.1" @@ -9838,11 +9892,12 @@ dependencies = [ [[package]] name = "opendal" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" dependencies = [ "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-retry", "opendal-layer-timeout", "opendal-service-azdls", @@ -9852,24 +9907,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "a43405d217dfdfb543f58847336d3af672897dd1939bb7dcf314b63cf364f1c9" dependencies = [ "anyhow", - "base64 0.22.1", + "asyncband", + "base64 0.23.1", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", - "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.3", "serde", "serde_json", "tokio", @@ -9878,11 +9931,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.5", +] + [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "06306202c97c54fb41bdbbdcb854c8798823f0022ae7aac7a40c8a1023aa83a6" dependencies = [ "backon", "log", @@ -9891,9 +9958,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "6bd334cbd0a0bc934146733e74a80a5faf8db8e014781a25f6d9d80d7b87c981" dependencies = [ "opendal-core", "tokio", @@ -9901,17 +9968,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "9064ed464286bffbea5955d470082a1e72b9b89f1e8f7a61b9e303c55ec08e4f" dependencies = [ - "base64 0.22.1", + "asyncband", + "base64 0.23.1", "bytes", "http 1.4.2", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -9921,9 +9989,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" dependencies = [ "http 1.4.2", "opendal-core", @@ -9931,9 +9999,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" dependencies = [ "async-trait", "bytes", @@ -9941,29 +10009,30 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", "serde", "serde_json", "tokio", + "uuid", ] [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "388b1d39b62535c62803754ebef89808859558697366dbedd0299345887ba461" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -10091,7 +10160,7 @@ dependencies = [ "bytes", "http 1.4.2", "opentelemetry", - "reqwest 0.13.3", + "reqwest 0.13.5", ] [[package]] @@ -10106,7 +10175,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.13.3", + "reqwest 0.13.5", "thiserror 2.0.18", "tokio", "tonic", @@ -10306,7 +10375,7 @@ dependencies = [ "libc", "redox_syscall 0.5.11", "smallvec", - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] @@ -11292,16 +11361,6 @@ dependencies = [ "serde", ] -[[package]] -name = "quick-xml" -version = "0.39.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.41.0" @@ -11321,6 +11380,63 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.5.10", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.5.10", + "tracing", + "windows-sys 0.59.0", +] + [[package]] name = "quote" version = "1.0.47" @@ -11477,6 +11593,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -11679,9 +11804,9 @@ dependencies = [ [[package]] name = "reqsign-aws-core" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d63b56638bb3cc7bd376a7cdce1ba3089777a08f47e4097888f2d784cc3f46c" +checksum = "bac4749b7dfa7bfaccd01eb03e9dc795ed37e3f20d6f0f38e2c67ee85ad6bc86" dependencies = [ "bytes", "form_urlencoded", @@ -11700,9 +11825,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0c499f4ed12d04c3d4c78fe4cb01aee22c9dae22848c14db2c6313d9df9f43" +checksum = "ff250f0fd0b913fbd565e405acc553da0f13bde30bfb5403178c9d0313cdc15f" dependencies = [ "bytes", "http 1.4.2", @@ -11715,9 +11840,9 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.1.2" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2824e7da3c2cc42ac3406c674eb57c89127fdcd97f3a73c608cfc680505ea134" +checksum = "c3f1fbc9add082f54e51e3bd9730f18d850a1f09d246baff5cd612c74ae4290e" dependencies = [ "anyhow", "base64 0.23.1", @@ -11725,7 +11850,7 @@ dependencies = [ "form_urlencoded", "http 1.4.2", "log", - "pem 3.0.6", + "pem 4.0.0", "percent-encoding", "reqsign-core", "rsa", @@ -11736,9 +11861,9 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.3.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ac1510872d9481205975d264deb39c109797e5068cc882ed9064270eaae5fa" +checksum = "ff052daffb0599681c50f85c59e7236438976efe991ab864edd9f3b235501a0f" dependencies = [ "anyhow", "base64 0.23.1", @@ -11749,6 +11874,7 @@ dependencies = [ "http 1.4.2", "jiff", "log", + "mea", "percent-encoding", "rsa", "serde", @@ -11760,9 +11886,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.5" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c3371bfc7e5c7f9627a04133af3583fd6c28715e7c83f79db38f3b384f535f" +checksum = "b3235df90a6bca681aa47dd86f2393d122a6d77042aa8a7c81e218cd45c5bfc0" dependencies = [ "anyhow", "reqsign-core", @@ -11771,9 +11897,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f81a9d38870892443489c0abb5332edfa81d5a14c437af9caef7c194897c92c1" +checksum = "272a5813571885c1455ffe106f0c768577e6ce69313446d4476e8d41dcf6ac5a" dependencies = [ "bytes", "form_urlencoded", @@ -11836,11 +11962,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-channel", "futures-core", @@ -11849,6 +11975,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.9.0", + "hyper-rustls", "hyper-tls 0.6.0", "hyper-util", "js-sys", @@ -11856,12 +11983,16 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower 0.5.3", "tower-http", @@ -12158,7 +12289,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.1", + "windows-sys 0.59.0", ] [[package]] @@ -12195,9 +12326,37 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.15" @@ -12426,7 +12585,7 @@ dependencies = [ "cfg_aliases", "httpdate", "native-tls", - "reqwest 0.13.3", + "reqwest 0.13.5", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -13331,7 +13490,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.1", + "windows-sys 0.59.0", ] [[package]] @@ -14876,9 +15035,9 @@ checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" [[package]] name = "windows-link" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-registry" @@ -14933,7 +15092,7 @@ version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f" dependencies = [ - "windows-link 0.2.0", + "windows-link 0.2.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 27422005f8eb2..00ed80ed4c1f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -676,9 +676,9 @@ async-compression = { git = "https://github.com/MaterializeInc/async-compression # NOTE: The `[workspace.dependencies]` version requirement above must stay # semver-compatible with this revision's crate version, otherwise Cargo drops # these patches into `[[patch.unused]]` and silently builds against crates.io. -iceberg = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "985929c6c5cc7083bac30c881e7ec43ec4d57b64" } -iceberg-catalog-rest = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "985929c6c5cc7083bac30c881e7ec43ec4d57b64" } -iceberg-storage-opendal = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "985929c6c5cc7083bac30c881e7ec43ec4d57b64" } +iceberg = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "eafda25053000154846871b36ed204e1bac03fb7" } +iceberg-catalog-rest = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "eafda25053000154846871b36ed204e1bac03fb7" } +iceberg-storage-opendal = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "eafda25053000154846871b36ed204e1bac03fb7" } # Custom duckdb crate to support mz needs # All changes should go to the `mz_changes` branch. diff --git a/deny.toml b/deny.toml index 0a72f4bde8c0a..8db75ab1b6671 100644 --- a/deny.toml +++ b/deny.toml @@ -138,7 +138,7 @@ skip = [ { name = "arrow-schema", version = "57.2.0" }, { name = "arrow-select", version = "57.2.0" }, { name = "arrow-string", version = "57.1.0" }, - # opendal 0.57 and reqsign 3.x (both via iceberg) are on the next generation + # opendal 0.59 and reqsign 3.x (both via iceberg) are on the next generation # of the RustCrypto crates, while the rest of the workspace is still on the # current one. { name = "base64", version = "0.23.1" }, @@ -150,6 +150,11 @@ skip = [ { name = "md-5", version = "0.11.0" }, { name = "sha1", version = "0.11.0" }, { name = "sha2", version = "0.11.0" }, + # reqsign-azure-storage 3.2 moved to pem 4, while jsonwebtoken, and so most + # of the workspace, is still on pem 3. The ADLS credential loader needs the + # `expires_at` field that 3.2 added to `Credential::SasToken`, so the older + # reqsign that would converge here is not an option. + { name = "pem", version = "4.0.0" }, # Held back by mz-timely-util; iceberg and the persist crates use 0.13. { name = "lz4_flex", version = "0.12.1" }, # Used by dynfmt; iceberg/typetag pulls in v0.4. @@ -251,6 +256,7 @@ wrappers = [ "reqsign-google", "reqwest", "rustls", + "rustls-platform-verifier", "sqlparser", "tokio-postgres", "tokio-tungstenite", diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index 036536d0abb62..82123671c853f 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -1224,13 +1224,13 @@ impl IcebergCatalogConnection { IcebergStorageProvider::Gcs => { Self::gcs_storage_factory(endpoint, &client, &token, &headers) } - // ADLS takes its credentials from the catalog's config, which - // `iceberg-rust` forwards to `opendal` the same way. OpenDAL's Azure - // service does not go through reqsign's credential provider - // abstraction, so there is no hook to wrap: vended credentials for - // ADLS work only through those static props, and stop working when - // they expire. - IcebergStorageProvider::Adls => OpenDalStorageFactory::Azdls, + // TODO(SS-449): ADLS can now take a loader too, so vended credentials + // here still expire without one. Wiring it needs a `VendedCredential` + // mapping from the catalog's `adls.sas-token.` prop onto + // `Credential::SasToken`. + IcebergStorageProvider::Adls => OpenDalStorageFactory::Azdls { + customized_credential_load: None, + }, }, // NOTE: We construct our own OAuth authenticator for the Catalog client instead of using the one built in. // This means we ignore auth overrides from `/v1/config` (e.g. `oauth2-server-uri`). From f9f4893dcc9954db0bfa49cc7e250c92c21b5249 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Wed, 16 Sep 2026 11:45:49 -0400 Subject: [PATCH 2/5] implement `VendedCredential` for `AzdlsCredential`, and include custom loader when building Azdls OpenDalStorageFactory --- src/storage-types/src/connections.rs | 15 ++++-- .../src/connections/iceberg_credentials.rs | 53 ++++++++++++++++++- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index 82123671c853f..de96672563e86 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -37,8 +37,8 @@ use iceberg_catalog_rest::{ RestCatalogBuilder, TokenProvider, }; use iceberg_storage_opendal::{ - AwsCredential, CustomAwsCredentialLoader, CustomGcsCredentialLoader, OpenDalStorageFactory, - ProvideCredential, + AwsCredential, CustomAwsCredentialLoader, CustomAzdlsCredentialLoader, + CustomGcsCredentialLoader, OpenDalStorageFactory, ProvideCredential, }; use itertools::Itertools; use mz_ccsr::tls::{Certificate, Identity}; @@ -1229,7 +1229,16 @@ impl IcebergCatalogConnection { // mapping from the catalog's `adls.sas-token.` prop onto // `Credential::SasToken`. IcebergStorageProvider::Adls => OpenDalStorageFactory::Azdls { - customized_credential_load: None, + customized_credential_load: endpoint.map(|endpoint| { + CustomAzdlsCredentialLoader::new( + iceberg_credentials::VendedCredentialLoader::new( + client.clone(), + endpoint, + Arc::clone(&token), + headers.clone(), + ), + ) + }), }, }, // NOTE: We construct our own OAuth authenticator for the Catalog client instead of using the one built in. diff --git a/src/storage-types/src/connections/iceberg_credentials.rs b/src/storage-types/src/connections/iceberg_credentials.rs index bc839385e8240..7a92b12ca59c2 100644 --- a/src/storage-types/src/connections/iceberg_credentials.rs +++ b/src/storage-types/src/connections/iceberg_credentials.rs @@ -30,7 +30,9 @@ use http::{HeaderMap, HeaderName, HeaderValue}; use iceberg::TableIdent; use iceberg::io::{GCS_TOKEN, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_SESSION_TOKEN}; use iceberg_catalog_rest::{StorageCredential, TokenProvider}; -use iceberg_storage_opendal::{AwsCredential, GcsCredential, GcsToken, ProvideCredential}; +use iceberg_storage_opendal::{ + AwsCredential, AzdlsCredential, GcsCredential, GcsToken, ProvideCredential, +}; use mz_ore::error::ErrorExt; use reqsign_core::time::Timestamp; use reqwest::StatusCode; @@ -121,6 +123,55 @@ impl VendedCredential for GcsCredential { } } +impl VendedCredential for AzdlsCredential { + const STORE: &'static str = "ADLS"; + + fn from_vended(vended: &StorageCredential) -> reqsign_core::Result { + let mut sas_token = None; + let mut expires_at = None; + for (prop, val) in vended.config.iter() { + if prop.starts_with("adls.sas-token.") { + if sas_token.is_some() { + return Err(reqsign_core::Error::credential_invalid(format!( + "vended Iceberg storage credential for prefix {} has multiple \ + {prop} properties", + vended.prefix + ))); + } + sas_token = Some(val.clone()); + } + if prop.starts_with("adls.sas-token-expires-at-ms.") { + if expires_at.is_some() { + return Err(reqsign_core::Error::credential_invalid(format!( + "vended Iceberg storage credential for prefix {} has multiple \ + {prop} properties", + vended.prefix + ))); + } + expires_at = vended_expires_at(vended, prop)?; + } + } + match (sas_token, expires_at) { + (Some(sas_token), Some(expires_at)) => Ok(AzdlsCredential::with_sas_token_expires_at( + &sas_token, expires_at, + )), + (Some(sas_token), None) => Ok(AzdlsCredential::with_sas_token(&sas_token)), + (None, _) => Err(reqsign_core::Error::credential_invalid(format!( + "vended Iceberg storage credential for prefix {} has no \ + adls.sas-token. property", + vended.prefix + ))), + } + } + + fn expires_at(&self) -> Option { + match self { + AzdlsCredential::SasToken { expires_at, .. } => *expires_at, + _ => None, + } + } +} + /// Reads a property a vended credential cannot do without. fn required(vended: &StorageCredential, prop: &str) -> reqsign_core::Result { vended.config.get(prop).cloned().ok_or_else(|| { From fde26562327495fd830c1fb007649baa0e0bc29f Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Wed, 16 Sep 2026 13:49:41 -0400 Subject: [PATCH 3/5] ignore additional `sas-token` props rather than error --- .../src/connections/iceberg_credentials.rs | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/storage-types/src/connections/iceberg_credentials.rs b/src/storage-types/src/connections/iceberg_credentials.rs index 7a92b12ca59c2..ba4c56984d217 100644 --- a/src/storage-types/src/connections/iceberg_credentials.rs +++ b/src/storage-types/src/connections/iceberg_credentials.rs @@ -131,14 +131,11 @@ impl VendedCredential for AzdlsCredential { let mut expires_at = None; for (prop, val) in vended.config.iter() { if prop.starts_with("adls.sas-token.") { - if sas_token.is_some() { - return Err(reqsign_core::Error::credential_invalid(format!( - "vended Iceberg storage credential for prefix {} has multiple \ - {prop} properties", - vended.prefix - ))); + // Some iceberg catalogs send multiple `adls.sas-token.` properties, with + // different suffixes. We just take the first one, and ignore the rest. + if sas_token.is_none() { + sas_token = Some(val.clone()); } - sas_token = Some(val.clone()); } if prop.starts_with("adls.sas-token-expires-at-ms.") { if expires_at.is_some() { @@ -155,7 +152,14 @@ impl VendedCredential for AzdlsCredential { (Some(sas_token), Some(expires_at)) => Ok(AzdlsCredential::with_sas_token_expires_at( &sas_token, expires_at, )), - (Some(sas_token), None) => Ok(AzdlsCredential::with_sas_token(&sas_token)), + (Some(sas_token), None) => { + warn!( + prefix = vended.prefix, + "vended Iceberg storage credential has a SAS token but no expiry; \ + re-fetching on a short interval" + ); + Ok(AzdlsCredential::with_sas_token(&sas_token)) + } (None, _) => Err(reqsign_core::Error::credential_invalid(format!( "vended Iceberg storage credential for prefix {} has no \ adls.sas-token. property", From d45e8927f9404bacd761bcbcf3708509a7957b08 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Wed, 16 Sep 2026 16:42:15 -0400 Subject: [PATCH 4/5] remove old comment --- src/storage-types/src/connections.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index de96672563e86..44d71a0cd008e 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -1224,10 +1224,6 @@ impl IcebergCatalogConnection { IcebergStorageProvider::Gcs => { Self::gcs_storage_factory(endpoint, &client, &token, &headers) } - // TODO(SS-449): ADLS can now take a loader too, so vended credentials - // here still expire without one. Wiring it needs a `VendedCredential` - // mapping from the catalog's `adls.sas-token.` prop onto - // `Credential::SasToken`. IcebergStorageProvider::Adls => OpenDalStorageFactory::Azdls { customized_credential_load: endpoint.map(|endpoint| { CustomAzdlsCredentialLoader::new( From 8e32108022f7f23fb94ddaa5d59a2a17af280220 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Wed, 16 Sep 2026 17:13:27 -0400 Subject: [PATCH 5/5] update revision after merging iceberg code --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27f43e01a9897..4d07124865631 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4967,7 +4967,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=eafda25053000154846871b36ed204e1bac03fb7#eafda25053000154846871b36ed204e1bac03fb7" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=9e252ad09e27950d2f3cbc7e2da0d1f5255731a0#9e252ad09e27950d2f3cbc7e2da0d1f5255731a0" dependencies = [ "aes-gcm", "anyhow", @@ -5024,7 +5024,7 @@ dependencies = [ [[package]] name = "iceberg-catalog-rest" version = "0.10.1" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=eafda25053000154846871b36ed204e1bac03fb7#eafda25053000154846871b36ed204e1bac03fb7" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=9e252ad09e27950d2f3cbc7e2da0d1f5255731a0#9e252ad09e27950d2f3cbc7e2da0d1f5255731a0" dependencies = [ "async-trait", "chrono", @@ -5043,7 +5043,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=eafda25053000154846871b36ed204e1bac03fb7#eafda25053000154846871b36ed204e1bac03fb7" +source = "git+https://github.com/MaterializeInc/iceberg-rust.git?rev=9e252ad09e27950d2f3cbc7e2da0d1f5255731a0#9e252ad09e27950d2f3cbc7e2da0d1f5255731a0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 00ed80ed4c1f9..526ab41cc33cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -676,9 +676,9 @@ async-compression = { git = "https://github.com/MaterializeInc/async-compression # NOTE: The `[workspace.dependencies]` version requirement above must stay # semver-compatible with this revision's crate version, otherwise Cargo drops # these patches into `[[patch.unused]]` and silently builds against crates.io. -iceberg = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "eafda25053000154846871b36ed204e1bac03fb7" } -iceberg-catalog-rest = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "eafda25053000154846871b36ed204e1bac03fb7" } -iceberg-storage-opendal = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "eafda25053000154846871b36ed204e1bac03fb7" } +iceberg = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "9e252ad09e27950d2f3cbc7e2da0d1f5255731a0" } +iceberg-catalog-rest = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "9e252ad09e27950d2f3cbc7e2da0d1f5255731a0" } +iceberg-storage-opendal = { git = "https://github.com/MaterializeInc/iceberg-rust.git", rev = "9e252ad09e27950d2f3cbc7e2da0d1f5255731a0" } # Custom duckdb crate to support mz needs # All changes should go to the `mz_changes` branch.