From 9023748384a0bb15d5749d5e5d4069f345b3d397 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Thu, 10 Sep 2026 16:38:46 -0400 Subject: [PATCH 1/8] chore: upgrade opendal to 0.59 Needed so that the ADLS backend can accept a custom credential provider: `AzdlsBuilder::credential_provider_chain` landed in opendal-service-azdls 0.59.1, and 0.57 offers no way to reach the signer. Note that ADLS *replaces* the provider chain, the way S3 does, rather than prepending to it the way GCS does, so an ADLS loader needs no equivalent of `suppress_default_credential_sources`. Two breaking changes come with the bump: * `Operator::new`/`from_config` return the operator directly, so the trailing `.finish()` is gone from all seven construction sites. * The HTTP transport is opt-in as of 0.59 (opendal RFC 7749). Without one, every request to an HTTP-backed service fails at call time with `ConfigInvalid` rather than failing to build, which showed up as the Glue schema-update tests failing to write table metadata. The workspace now enables `http-transport-reqwest` and `create_operator` installs the default transport behind a `Once`, rather than relying on opendal's pre-`main` constructor, which its own documentation warns linkers may drop when opendal is linked as a `staticlib` for the Python bindings. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 611 +++++++++++------- Cargo.toml | 5 +- crates/storage/opendal/public-api.txt | 2 +- crates/storage/opendal/src/azdls.rs | 4 +- crates/storage/opendal/src/fs.rs | 4 +- crates/storage/opendal/src/gcs.rs | 2 +- crates/storage/opendal/src/hf.rs | 4 +- crates/storage/opendal/src/lib.rs | 18 + crates/storage/opendal/src/memory.rs | 4 +- crates/storage/opendal/src/oss.rs | 2 +- crates/storage/opendal/src/s3.rs | 2 +- .../storage/opendal/tests/file_io_gcs_test.rs | 3 +- 12 files changed, 400 insertions(+), 261 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0f38f36c1..1ceed693da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,7 +205,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -216,7 +216,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -370,7 +370,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -491,7 +491,7 @@ version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" dependencies = [ - "bitflags", + "bitflags 2.11.0", "serde_core", "serde_json", ] @@ -621,6 +621,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + [[package]] name = "atoi" version = "2.0.0" @@ -664,7 +674,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.0", - "sha1", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -691,7 +701,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", - "untrusted 0.7.1", "zeroize", ] @@ -866,11 +875,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -1067,6 +1076,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1103,6 +1118,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.0" @@ -1188,7 +1209,7 @@ version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -1440,6 +1461,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -1654,12 +1681,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1753,20 +1781,14 @@ dependencies = [ [[package]] name = "ctor" -version = "0.6.3" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" dependencies = [ - "ctor-proc-macro", - "dtor", + "link-section", + "linktime-proc-macro", ] -[[package]] -name = "ctor-proc-macro" -version = "0.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" - [[package]] name = "ctr" version = "0.9.2" @@ -1776,6 +1798,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "darling" version = "0.20.11" @@ -2289,7 +2320,7 @@ checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -2308,7 +2339,7 @@ dependencies = [ "num-traits", "rand 0.9.4", "regex", - "sha2", + "sha2 0.10.9", "unicode-segmentation", "uuid", ] @@ -2647,8 +2678,8 @@ dependencies = [ "percent-encoding", "rand 0.9.4", "serde_json", - "sha1", - "sha2", + "sha1 0.10.6", + "sha2 0.10.9", "url", ] @@ -2717,6 +2748,37 @@ dependencies = [ "url", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "der" version = "0.7.10" @@ -2796,6 +2858,7 @@ dependencies = [ "block-buffer 0.12.0", "const-oid 0.10.2", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -2816,7 +2879,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -2857,21 +2920,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" -[[package]] -name = "dtor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301" -dependencies = [ - "dtor-proc-macro", -] - -[[package]] -name = "dtor-proc-macro" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" - [[package]] name = "dunce" version = "1.0.5" @@ -2984,7 +3032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -3100,7 +3148,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags", + "bitflags 2.11.0", "rustc_version", ] @@ -3123,7 +3171,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.8", ] [[package]] @@ -3514,18 +3562,21 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hf-xet" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430b33fa84f92796d4d263070b6c0d3ca219df7b9a0e1853ee431029b1612bcd" +checksum = "c237ef4fb0ce1962a5117f8bd8c74454b41629826a9df17d14a1840ca18f0754" dependencies = [ + "anyhow", "async-trait", "bytes", "http 1.4.0", "more-asserts", "serde", + "serde_json", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "uuid", "xet-client", @@ -3552,7 +3603,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -3564,6 +3615,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "home" version = "0.5.11" @@ -3717,7 +3777,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3728,7 +3788,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -3780,7 +3840,7 @@ dependencies = [ "as-any", "async-trait", "backon", - "base64", + "base64 0.22.1", "bimap", "bytes", "chrono", @@ -4295,10 +4355,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -4307,15 +4369,25 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.60.2", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", ] [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", "syn 2.0.117", @@ -4405,23 +4477,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonwebtoken" -version = "10.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" -dependencies = [ - "aws-lc-rs", - "base64", - "getrandom 0.2.17", - "js-sys", - "pem", - "serde", - "serde_json", - "signature", - "simple_asn1", -] - [[package]] name = "konst" version = "0.4.3" @@ -4445,7 +4500,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -4575,7 +4630,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", "plain", "redox_syscall 0.7.3", @@ -4604,6 +4659,12 @@ dependencies = [ "escape8259", ] +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -4621,6 +4682,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -4644,9 +4711,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru-slab" @@ -4709,10 +4776,11 @@ dependencies = [ [[package]] name = "mea" -version = "0.6.3" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832" +checksum = "c709842c4ce65cb91e2666ad5319dfc1efc3af0d34f02075eddca9000d9f8afb" dependencies = [ + "hashbrown 0.17.1", "slab", ] @@ -4960,7 +5028,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if 1.0.4", "cfg_aliases", "libc", @@ -4973,7 +5041,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if 1.0.4", "cfg_aliases", "libc", @@ -4994,7 +5062,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -5097,7 +5165,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -5135,7 +5203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -5150,7 +5218,7 @@ dependencies = [ "md-5 0.10.6", "parking_lot", "percent-encoding", - "quick-xml", + "quick-xml 0.39.4", "rand 0.10.1", "reqwest 0.12.28", "ring", @@ -5193,11 +5261,12 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" dependencies = [ "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-retry", "opendal-layer-timeout", "opendal-service-azdls", @@ -5210,24 +5279,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "a43405d217dfdfb543f58847336d3af672897dd1939bb7dcf314b63cf364f1c9" dependencies = [ "anyhow", - "base64", + "asyncband", + "base64 0.23.1", "bytes", "futures", "http 1.4.0", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", - "mea", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.3", "serde", "serde_json", "tokio", @@ -5236,11 +5303,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06" +dependencies = [ + "bytes", + "futures", + "http 1.4.0", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.5", +] + [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "06306202c97c54fb41bdbbdcb854c8798823f0022ae7aac7a40c8a1023aa83a6" dependencies = [ "backon", "log", @@ -5249,9 +5330,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "6bd334cbd0a0bc934146733e74a80a5faf8db8e014781a25f6d9d80d7b87c981" dependencies = [ "opendal-core", "tokio", @@ -5259,17 +5340,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "9064ed464286bffbea5955d470082a1e72b9b89f1e8f7a61b9e303c55ec08e4f" dependencies = [ - "base64", + "asyncband", + "base64 0.23.1", "bytes", "http 1.4.0", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5279,9 +5361,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" dependencies = [ "http 1.4.0", "opendal-core", @@ -5289,9 +5371,9 @@ dependencies = [ [[package]] name = "opendal-service-fs" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e89a665fef0e6bd249cf5ea47fc174b7ba892159bee4b9382528b1ca873a2c" +checksum = "fb9caf04d6d38713299dd4abac984b95ab16ee20e1ff09160f495c5a64644083" dependencies = [ "bytes", "log", @@ -5303,9 +5385,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" dependencies = [ "async-trait", "bytes", @@ -5313,43 +5395,44 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", "serde", "serde_json", "tokio", + "uuid", ] [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "5c17b59cf22bd2da9f751b8e66db595d5fe546b4c6b6ff0fb84c7bfd27455d7a" dependencies = [ + "asyncband", "bytes", "hf-xet", "http 1.4.0", "log", "opendal-core", "percent-encoding", - "reqwest 0.13.3", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "284373c4a1143d8efaa7d010c1db05856d33a7cad475aaee8405dbcb0660cd96" dependencies = [ "bytes", "http 1.4.0", "log", "opendal-core", - "quick-xml", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5358,18 +5441,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "388b1d39b62535c62803754ebef89808859558697366dbedd0299345887ba461" dependencies = [ - "base64", + "base64 0.23.1", "bytes", - "crc32c", + "crc-fast", "http 1.4.0", "log", "md-5 0.11.0", "opendal-core", - "quick-xml", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5383,7 +5466,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if 1.0.4", "foreign-types", "libc", @@ -5519,7 +5602,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -5555,7 +5638,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -5593,16 +5676,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", - "hmac", + "hmac 0.12.1", ] [[package]] name = "pem" -version = "3.0.6" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" dependencies = [ - "base64", + "base64 0.23.1", "serde_core", ] @@ -5727,7 +5810,7 @@ dependencies = [ "der", "pbkdf2", "scrypt", - "sha2", + "sha2 0.10.9", "spki", ] @@ -5887,7 +5970,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5906,7 +5989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -6038,6 +6121,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6051,7 +6144,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -6089,9 +6182,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6247,7 +6340,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -6256,7 +6349,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -6358,9 +6451,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.0.0" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95" +checksum = "10909dbc9fcb78f913014d78a5c9d1d51c477f491b1006fd1baba4fd286181df" dependencies = [ "anyhow", "form_urlencoded", @@ -6374,38 +6467,52 @@ dependencies = [ ] [[package]] -name = "reqsign-aws-v4" -version = "3.0.0" +name = "reqsign-aws-core" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43" +checksum = "bac4749b7dfa7bfaccd01eb03e9dc795ed37e3f20d6f0f38e2c67ee85ad6bc86" dependencies = [ - "anyhow", "bytes", "form_urlencoded", + "hex", "http 1.4.0", "log", "percent-encoding", - "quick-xml", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", "serde_json", "serde_urlencoded", - "sha1", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff250f0fd0b913fbd565e405acc553da0f13bde30bfb5403178c9d0313cdc15f" +dependencies = [ + "bytes", + "http 1.4.0", + "log", + "quick-xml 0.41.0", + "reqsign-aws-core", + "reqsign-core", + "serde", ] [[package]] name = "reqsign-azure-storage" -version = "3.0.0" +version = "3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6" +checksum = "c3f1fbc9add082f54e51e3bd9730f18d850a1f09d246baff5cd612c74ae4290e" dependencies = [ "anyhow", - "base64", + "base64 0.23.1", "bytes", "form_urlencoded", "http 1.4.0", - "jsonwebtoken", "log", "pem", "percent-encoding", @@ -6413,36 +6520,39 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha1", + "sha1 0.11.0", ] [[package]] name = "reqsign-core" -version = "3.0.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102" +checksum = "ff052daffb0599681c50f85c59e7236438976efe991ab864edd9f3b235501a0f" dependencies = [ "anyhow", - "base64", + "base64 0.23.1", "bytes", - "form_urlencoded", "futures", "hex", - "hmac", + "hmac 0.13.0", "http 1.4.0", "jiff", "log", + "mea", "percent-encoding", - "sha1", - "sha2", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", "windows-sys 0.61.2", ] [[package]] name = "reqsign-file-read-tokio" -version = "3.0.0" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5" +checksum = "b3235df90a6bca681aa47dd86f2393d122a6d77042aa8a7c81e218cd45c5bfc0" dependencies = [ "anyhow", "reqsign-core", @@ -6451,13 +6561,13 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e" +checksum = "272a5813571885c1455ffe106f0c768577e6ce69313446d4476e8d41dcf6ac5a" dependencies = [ + "bytes", "form_urlencoded", "http 1.4.0", - "jsonwebtoken", "log", "percent-encoding", "reqsign-aws-v4", @@ -6465,7 +6575,6 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2", "tokio", ] @@ -6475,7 +6584,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -6516,11 +6625,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-core", "futures-util", @@ -6563,7 +6672,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.4.0", - "reqwest 0.13.3", + "reqwest 0.13.5", "thiserror 2.0.18", "tower-service", ] @@ -6578,7 +6687,7 @@ dependencies = [ "cfg-if 1.0.4", "getrandom 0.2.17", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] @@ -6635,7 +6744,7 @@ dependencies = [ "pkcs1", "pkcs8", "rand_core 0.6.4", - "sha2", + "sha2 0.10.9", "signature", "spki", "subtle", @@ -6705,11 +6814,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6767,7 +6876,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6785,7 +6894,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -6800,7 +6909,7 @@ version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if 1.0.4", "clipboard-win", "fd-lock", @@ -6917,7 +7026,7 @@ checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ "pbkdf2", "salsa20", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -6926,7 +7035,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -7097,7 +7206,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -7147,6 +7256,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -7156,16 +7276,17 @@ dependencies = [ "cfg-if 1.0.4", "cpufeatures 0.2.17", "digest 0.10.7", - "sha2-asm", ] [[package]] -name = "sha2-asm" -version = "0.6.4" +name = "sha2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b845214d6175804686b2bd482bcffe96651bb2d1200742b712003504a2dac1ab" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ - "cc", + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -7242,18 +7363,6 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" -[[package]] -name = "simple_asn1" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror 2.0.18", - "time", -] - [[package]] name = "siphasher" version = "1.0.2" @@ -7298,7 +7407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -7348,6 +7457,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -7424,7 +7539,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "crc", "crossbeam-queue", @@ -7444,7 +7559,7 @@ dependencies = [ "rustls", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "thiserror 2.0.18", "tokio", @@ -7482,7 +7597,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "sqlx-core", "sqlx-sqlite", "syn 2.0.117", @@ -7497,8 +7612,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", - "bitflags", + "base64 0.22.1", + "bitflags 2.11.0", "byteorder", "bytes", "crc", @@ -7512,7 +7627,7 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", "md-5 0.10.6", @@ -7521,8 +7636,8 @@ dependencies = [ "percent-encoding", "rand 0.8.5", "rsa", - "sha1", - "sha2", + "sha1 0.10.6", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -7538,8 +7653,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", - "bitflags", + "base64 0.22.1", + "bitflags 2.11.0", "byteorder", "crc", "dotenvy", @@ -7549,7 +7664,7 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", @@ -7559,7 +7674,7 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", "sqlx-core", "stringprep", @@ -7774,7 +7889,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -7811,7 +7926,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -8029,6 +8144,30 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio_with_wasm" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6" +dependencies = [ + "js-sys", + "tokio", + "tokio_with_wasm_proc", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "tokio_with_wasm_proc" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "toml" version = "0.8.23" @@ -8121,7 +8260,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.11.0", "bytes", "futures-util", "http 1.4.0", @@ -8448,12 +8587,6 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -8762,7 +8895,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap 2.13.0", "semver", @@ -9297,7 +9430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.0", "indexmap 2.13.0", "log", "serde", @@ -9345,24 +9478,22 @@ dependencies = [ [[package]] name = "xet-client" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" +checksum = "c3b8da8cc70aa2e3c500c0400e012df82c656ab9fca47f9f939fffc5afd89aca" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", - "clap", "crc32fast", "futures", "http 1.4.0", "hyper", - "lazy_static", "more-asserts", "rand 0.10.1", "redb", - "reqwest 0.13.3", + "reqwest 0.13.5", "reqwest-middleware", "serde", "serde_json", @@ -9372,8 +9503,8 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-retry", + "tokio_with_wasm", "tracing", - "tracing-subscriber", "url", "urlencoding", "web-time", @@ -9383,24 +9514,21 @@ dependencies = [ [[package]] name = "xet-core-structures" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" +checksum = "73503c223783dccc864abde22115e09d12f190448a0baf58ab2c54bc709e2f99" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", - "clap", "countio", - "csv", "futures", "futures-util", "getrandom 0.4.2", "heapify", "itertools 0.14.0", - "lazy_static", "lz4_flex", "more-asserts", "rand 0.10.1", @@ -9408,7 +9536,6 @@ dependencies = [ "safe-transmute", "serde", "static_assertions", - "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", @@ -9420,32 +9547,31 @@ dependencies = [ [[package]] name = "xet-data" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fd409bef621411a9d9013798540bb8036cb2678f03ab39af89a5e88034ed8c" +checksum = "c89052ec5dec2187cad30b86af92cc24fd61c4a57a795f1ff7ff5f38d49184eb" dependencies = [ "anyhow", "async-trait", "bytes", "chrono", - "clap", "gearhash", "http 1.4.0", "itertools 0.14.0", - "lazy_static", "more-asserts", "rand 0.10.1", "serde", "serde_json", - "sha2", + "sha2 0.11.0", "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "url", "uuid", - "walkdir", + "web-time", "xet-client", "xet-core-structures", "xet-runtime", @@ -9453,9 +9579,9 @@ dependencies = [ [[package]] name = "xet-runtime" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d8f121c33866f7648b737abe70d0e2dd9c0af4ffdd7219207531d0283aa63d" +checksum = "af5c60d5eed38ab4c576f4421bae835e7bd07631fb381705605529d2015c106b" dependencies = [ "anyhow", "async-trait", @@ -9469,13 +9595,12 @@ dependencies = [ "git-version", "humantime", "konst", - "lazy_static", "libc", "more-asserts", "oneshot", "pin-project", "rand 0.10.1", - "reqwest 0.13.3", + "reqwest 0.13.5", "serde", "serde_json", "shellexpand", @@ -9483,9 +9608,11 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tokio-util", + "tokio_with_wasm", "tracing", "tracing-appender", "tracing-subscriber", + "web-time", "whoami 2.1.1", "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index c2b9e80869..d7233d10c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,8 +117,11 @@ mockito = "1" motore-macros = "0.4.3" murmur3 = "0.5.2" once_cell = "1.20" -opendal = { version = "0.57", default-features = false, features = [ +opendal = { version = "0.59", default-features = false, features = [ "executors-tokio", + # 0.59 made the HTTP transport opt-in. `iceberg-storage-opendal` installs it, + # since every object-store backend is unusable without one. + "http-transport-reqwest", "services-memory", # `iceberg-storage-opendal` wraps every operator in these two layers. "layers-retry", diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index 39e438f931..6815399781 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -60,7 +60,7 @@ impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalS pub fn iceberg_storage_opendal::OpenDalStorageFactory::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg_storage_opendal::CustomAwsCredentialLoader(_) impl iceberg_storage_opendal::CustomAwsCredentialLoader -pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::new(provider: impl reqsign_core::api::ProvideCredential + 'static) -> Self +pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::new(provider: impl reqsign_core::api::ProvideCredential + 'static) -> Self impl core::clone::Clone for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::clone(&self) -> Self impl core::fmt::Debug for iceberg_storage_opendal::CustomAwsCredentialLoader diff --git a/crates/storage/opendal/src/azdls.rs b/crates/storage/opendal/src/azdls.rs index b47c55d9e7..6d39320df8 100644 --- a/crates/storage/opendal/src/azdls.rs +++ b/crates/storage/opendal/src/azdls.rs @@ -201,9 +201,7 @@ fn azdls_config_build(config: &AzdlsConfig, path: &AzureStoragePath) -> Result Result { let mut cfg = FsConfig::default(); cfg.root = Some("/".to_string()); - Ok(Operator::from_config(cfg) - .map_err(from_opendal_error)? - .finish()) + Operator::from_config(cfg).map_err(from_opendal_error) } diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index 2fc4e8a528..b4b1e6b81c 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -131,7 +131,7 @@ pub(crate) fn gcs_config_build( } }; - Ok(Operator::new(builder).map_err(from_opendal_error)?.finish()) + Operator::new(builder).map_err(from_opendal_error) } /// Custom GCS credential loader. diff --git a/crates/storage/opendal/src/hf.rs b/crates/storage/opendal/src/hf.rs index a7ca2d884d..d38229cedf 100644 --- a/crates/storage/opendal/src/hf.rs +++ b/crates/storage/opendal/src/hf.rs @@ -197,9 +197,7 @@ pub(crate) fn hf_config_build<'a>( .ok_or_else(|| Error::new(ErrorKind::DataInvalid, format!("Invalid hf url: {path}")))?; let relative_path = &path[path.len() - parsed.path.len()..]; - let op = Operator::from_config(hf_cfg) - .map_err(from_opendal_error)? - .finish(); + let op = Operator::from_config(hf_cfg).map_err(from_opendal_error)?; Ok((op, relative_path)) } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index d69868befb..04b54ff8cc 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -264,6 +264,22 @@ pub enum OpenDalStorage { }, } +/// Installs OpenDAL's default HTTP transport, once per process. +/// +/// As of OpenDAL 0.59 the transport is opt-in: without one, every request to an HTTP-backed +/// service fails with `ConfigInvalid` rather than at construction time. OpenDAL's own +/// pre-`main` constructor would install it, but only under `auto-register-services`, and its +/// documentation warns that linkers may drop that constructor when OpenDAL is linked as a +/// `staticlib`, which is how the Python bindings consume it. Installing here instead keeps the +/// behavior identical across every consumer. +/// +/// `install_default` is first-installed-wins, so an application that installs its own transport +/// before touching a `FileIO` keeps it. +fn install_http_transport() { + static INSTALL: std::sync::Once = std::sync::Once::new(); + INSTALL.call_once(opendal::install_default); +} + impl OpenDalStorage { /// Creates operator from path. /// @@ -282,6 +298,8 @@ impl OpenDalStorage { &self, path: &'a impl AsRef, ) -> Result<(Operator, &'a str)> { + install_http_transport(); + let path = path.as_ref(); let (operator, relative_path): (Operator, &str) = match self { #[cfg(feature = "opendal-memory")] diff --git a/crates/storage/opendal/src/memory.rs b/crates/storage/opendal/src/memory.rs index 5957d0e848..746e927161 100644 --- a/crates/storage/opendal/src/memory.rs +++ b/crates/storage/opendal/src/memory.rs @@ -22,7 +22,5 @@ use opendal::services::MemoryConfig; use crate::utils::from_opendal_error; pub(crate) fn memory_config_build() -> Result { - Ok(Operator::from_config(MemoryConfig::default()) - .map_err(from_opendal_error)? - .finish()) + Operator::from_config(MemoryConfig::default()).map_err(from_opendal_error) } diff --git a/crates/storage/opendal/src/oss.rs b/crates/storage/opendal/src/oss.rs index add8b7a0f7..a1cf44549c 100644 --- a/crates/storage/opendal/src/oss.rs +++ b/crates/storage/opendal/src/oss.rs @@ -53,5 +53,5 @@ pub(crate) fn oss_config_build(cfg: &OssConfig, path: &str) -> Result let builder = cfg.clone().into_builder().bucket(bucket); - Ok(Operator::new(builder).map_err(from_opendal_error)?.finish()) + Operator::new(builder).map_err(from_opendal_error) } diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index cbb450e999..2d0c4c1b42 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -148,7 +148,7 @@ pub(crate) fn s3_config_build( builder = builder.credential_provider_chain(chain); } - Ok(Operator::new(builder).map_err(from_opendal_error)?.finish()) + Operator::new(builder).map_err(from_opendal_error) } /// Custom AWS credential loader. diff --git a/crates/storage/opendal/tests/file_io_gcs_test.rs b/crates/storage/opendal/tests/file_io_gcs_test.rs index c1cbeb8383..42188512fa 100644 --- a/crates/storage/opendal/tests/file_io_gcs_test.rs +++ b/crates/storage/opendal/tests/file_io_gcs_test.rs @@ -215,8 +215,7 @@ mod tests { let chain = ProvideCredentialChain::new().push(MockCredentialLoader(None)); let operator = Operator::new(cfg.into_builder().credential_provider_chain(chain)) - .expect("operator builds") - .finish(); + .expect("operator builds"); assert!( !operator.exists("any").await.expect("the request is signed"), From 92d465c9b4d4c470c38787fdc050b9bab9919e44 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Thu, 10 Sep 2026 16:51:35 -0400 Subject: [PATCH 2/8] add azdls custom credential loader to opendal storage builder --- Cargo.lock | 1 + crates/storage/opendal/Cargo.toml | 3 +- crates/storage/opendal/public-api.txt | 34 +++++++++++++++++ crates/storage/opendal/src/azdls.rs | 51 +++++++++++++++++++++++-- crates/storage/opendal/src/lib.rs | 30 ++++++++++++--- crates/storage/opendal/src/resolving.rs | 13 +++++++ 6 files changed, 122 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ceed693da..703bc01d74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4105,6 +4105,7 @@ dependencies = [ "iceberg_test_utils", "opendal", "reqsign-aws-v4", + "reqsign-azure-storage", "reqsign-core", "reqsign-google", "reqwest 0.12.28", diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index 48f6b106f6..d650bc407c 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -39,7 +39,7 @@ opendal-all = [ "opendal-hf", ] -opendal-azdls = ["opendal/services-azdls"] +opendal-azdls = ["opendal/services-azdls", "reqsign-azure-storage", "reqsign-core"] opendal-fs = ["opendal/services-fs"] opendal-gcs = ["opendal/services-gcs", "reqsign-google", "reqsign-core"] opendal-hf = ["opendal/services-hf"] @@ -56,6 +56,7 @@ futures = { workspace = true } iceberg = { workspace = true } opendal = { workspace = true } reqsign-aws-v4 = { version = "3.0.0", optional = true } +reqsign-azure-storage = { version = "3.2.1", optional = true } reqsign-core = { version = "3.0.0", optional = true } reqsign-google = { version = "3.0.0", optional = true } serde = { workspace = true } diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index 6815399781..b42894a9f4 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -1,11 +1,36 @@ pub mod iceberg_storage_opendal pub use iceberg_storage_opendal::AwsCredential +pub use iceberg_storage_opendal::AzdlsCredential pub use iceberg_storage_opendal::GcsCredential pub use iceberg_storage_opendal::GcsToken pub use iceberg_storage_opendal::ProvideCredential +pub enum iceberg_storage_opendal::AzureStorageScheme +pub iceberg_storage_opendal::AzureStorageScheme::Abfs +pub iceberg_storage_opendal::AzureStorageScheme::Abfss +pub iceberg_storage_opendal::AzureStorageScheme::Wasb +pub iceberg_storage_opendal::AzureStorageScheme::Wasbs +impl iceberg_storage_opendal::AzureStorageScheme +pub fn iceberg_storage_opendal::AzureStorageScheme::as_http_scheme(&self) -> &str +impl core::clone::Clone for iceberg_storage_opendal::AzureStorageScheme +pub fn iceberg_storage_opendal::AzureStorageScheme::clone(&self) -> iceberg_storage_opendal::AzureStorageScheme +impl core::cmp::PartialEq for iceberg_storage_opendal::AzureStorageScheme +pub fn iceberg_storage_opendal::AzureStorageScheme::eq(&self, other: &iceberg_storage_opendal::AzureStorageScheme) -> bool +impl core::fmt::Debug for iceberg_storage_opendal::AzureStorageScheme +pub fn iceberg_storage_opendal::AzureStorageScheme::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for iceberg_storage_opendal::AzureStorageScheme +pub fn iceberg_storage_opendal::AzureStorageScheme::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg_storage_opendal::AzureStorageScheme +impl core::str::traits::FromStr for iceberg_storage_opendal::AzureStorageScheme +pub type iceberg_storage_opendal::AzureStorageScheme::Err = iceberg::error::Error +pub fn iceberg_storage_opendal::AzureStorageScheme::from_str(s: &str) -> iceberg::error::Result +impl serde_core::ser::Serialize for iceberg_storage_opendal::AzureStorageScheme +pub fn iceberg_storage_opendal::AzureStorageScheme::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::AzureStorageScheme +pub fn iceberg_storage_opendal::AzureStorageScheme::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub enum iceberg_storage_opendal::OpenDalStorage pub iceberg_storage_opendal::OpenDalStorage::Azdls pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::Azdls::customized_credential_load: core::option::Option pub iceberg_storage_opendal::OpenDalStorage::Gcs pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::Gcs::customized_credential_load: core::option::Option @@ -40,6 +65,7 @@ impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::OpenDalS pub fn iceberg_storage_opendal::OpenDalStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub enum iceberg_storage_opendal::OpenDalStorageFactory pub iceberg_storage_opendal::OpenDalStorageFactory::Azdls +pub iceberg_storage_opendal::OpenDalStorageFactory::Azdls::customized_credential_load: core::option::Option pub iceberg_storage_opendal::OpenDalStorageFactory::Fs pub iceberg_storage_opendal::OpenDalStorageFactory::Gcs pub iceberg_storage_opendal::OpenDalStorageFactory::Gcs::customized_credential_load: core::option::Option @@ -65,6 +91,13 @@ impl core::clone::Clone for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::clone(&self) -> Self impl core::fmt::Debug for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_storage_opendal::CustomAzdlsCredentialLoader(_) +impl iceberg_storage_opendal::CustomAzdlsCredentialLoader +pub fn iceberg_storage_opendal::CustomAzdlsCredentialLoader::new(provider: impl reqsign_core::api::ProvideCredential + 'static) -> Self +impl core::clone::Clone for iceberg_storage_opendal::CustomAzdlsCredentialLoader +pub fn iceberg_storage_opendal::CustomAzdlsCredentialLoader::clone(&self) -> Self +impl core::fmt::Debug for iceberg_storage_opendal::CustomAzdlsCredentialLoader +pub fn iceberg_storage_opendal::CustomAzdlsCredentialLoader::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg_storage_opendal::CustomGcsCredentialLoader(_) impl iceberg_storage_opendal::CustomGcsCredentialLoader pub fn iceberg_storage_opendal::CustomGcsCredentialLoader::new(provider: impl reqsign_core::api::ProvideCredential + 'static) -> Self @@ -94,6 +127,7 @@ pub fn iceberg_storage_opendal::OpenDalResolvingStorage::deserialize<__D>(__dese pub struct iceberg_storage_opendal::OpenDalResolvingStorageFactory impl iceberg_storage_opendal::OpenDalResolvingStorageFactory pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::new() -> Self +pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::with_azdls_credential_loader(self, loader: iceberg_storage_opendal::CustomAzdlsCredentialLoader) -> Self pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::with_gcs_credential_loader(self, loader: iceberg_storage_opendal::CustomGcsCredentialLoader) -> Self pub fn iceberg_storage_opendal::OpenDalResolvingStorageFactory::with_s3_credential_loader(self, loader: iceberg_storage_opendal::CustomAwsCredentialLoader) -> Self impl core::clone::Clone for iceberg_storage_opendal::OpenDalResolvingStorageFactory diff --git a/crates/storage/opendal/src/azdls.rs b/crates/storage/opendal/src/azdls.rs index 6d39320df8..55c5c66e6f 100644 --- a/crates/storage/opendal/src/azdls.rs +++ b/crates/storage/opendal/src/azdls.rs @@ -18,6 +18,7 @@ use std::collections::HashMap; use std::fmt::Display; use std::str::FromStr; +use std::sync::Arc; use iceberg::io::{ ADLS_ACCOUNT_KEY, ADLS_ACCOUNT_NAME, ADLS_AUTHORITY_HOST, ADLS_CLIENT_ID, ADLS_CLIENT_SECRET, @@ -26,6 +27,8 @@ use iceberg::io::{ use iceberg::{Error, ErrorKind, Result}; use opendal::Configurator; use opendal::services::AzdlsConfig; +pub use reqsign_azure_storage::Credential as AzdlsCredential; +use reqsign_core::{ProvideCredential, ProvideCredentialChain, ProvideCredentialDyn}; use serde::{Deserialize, Serialize}; use url::Url; @@ -90,12 +93,13 @@ pub(crate) fn azdls_config_parse(mut properties: HashMap) -> Res /// `abfss://@.dfs.core.windows.net/mydir/myfile.parquet`. pub(crate) fn azdls_create_operator<'a>( absolute_path: &'a str, + customized_credential_load: &Option, config: &AzdlsConfig, ) -> Result<(opendal::Operator, &'a str)> { let path = absolute_path.parse::()?; match_path_with_config(&path, config)?; - let op = azdls_config_build(config, &path)?; + let op = azdls_config_build(config, customized_credential_load, &path)?; // Paths to files in ADLS tend to be written in fully qualified form, // including their filesystem and account name. @@ -192,7 +196,11 @@ pub(crate) fn match_path_with_config(path: &AzureStoragePath, config: &AzdlsConf Ok(()) } -fn azdls_config_build(config: &AzdlsConfig, path: &AzureStoragePath) -> Result { +fn azdls_config_build( + config: &AzdlsConfig, + customized_credential_load: &Option, + path: &AzureStoragePath, +) -> Result { let mut builder = config.clone().into_builder(); if config.endpoint.is_none() { @@ -200,6 +208,10 @@ fn azdls_config_build(config: &AzdlsConfig, path: &AzureStoragePath) -> Result>); + +impl Clone for CustomAzdlsCredentialLoader { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } +} + +impl std::fmt::Debug for CustomAzdlsCredentialLoader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CustomAzdlsCredentialLoader") + .finish_non_exhaustive() + } +} + +impl CustomAzdlsCredentialLoader { + /// Create a new custom AZDLS credential loader from any [`ProvideCredential`] implementation. + pub fn new(provider: impl ProvideCredential + 'static) -> Self { + Self(Arc::new(provider) + as Arc< + dyn ProvideCredentialDyn, + >) + } +} + #[cfg(test)] mod tests { use std::collections::HashMap; @@ -464,7 +509,7 @@ mod tests { ]; for (name, input, expected) in test_cases { - let result = azdls_create_operator(input.0, &input.1); + let result = azdls_create_operator(input.0, &None, &input.1); match expected { Some((expected_filesystem, expected_path)) => { assert!(result.is_ok(), "Test case {name} failed: {result:?}"); diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 04b54ff8cc..73b14c716f 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -46,7 +46,7 @@ use utils::from_opendal_error; cfg_if! { if #[cfg(feature = "opendal-azdls")] { mod azdls; - use azdls::*; + pub use azdls::*; use opendal::services::AzdlsConfig; } } @@ -102,7 +102,11 @@ cfg_if! { /// /// Downstream implementors must name the trait through this re-export: a loader built /// against a differently-versioned `reqsign-core` will not satisfy the bound. -#[cfg(any(feature = "opendal-s3", feature = "opendal-gcs"))] +#[cfg(any( + feature = "opendal-s3", + feature = "opendal-gcs", + feature = "opendal-azdls" +))] pub use reqsign_core::ProvideCredential; mod resolving; @@ -139,7 +143,11 @@ pub enum OpenDalStorageFactory { Oss, /// Azure Data Lake Storage factory. #[cfg(feature = "opendal-azdls")] - Azdls, + Azdls { + /// Custom AZDLS credential loader. + #[serde(skip)] + customized_credential_load: Option, + }, /// HuggingFace Hub storage factory. #[cfg(feature = "opendal-hf")] Hf, @@ -175,8 +183,11 @@ impl StorageFactory for OpenDalStorageFactory { config: oss_config_parse(config.props().clone())?.into(), })), #[cfg(feature = "opendal-azdls")] - OpenDalStorageFactory::Azdls => Ok(Arc::new(OpenDalStorage::Azdls { + OpenDalStorageFactory::Azdls { + customized_credential_load, + } => Ok(Arc::new(OpenDalStorage::Azdls { config: azdls_config_parse(config.props().clone())?.into(), + customized_credential_load: customized_credential_load.clone(), })), #[cfg(feature = "opendal-hf")] OpenDalStorageFactory::Hf => Ok(Arc::new(OpenDalStorage::Hf { @@ -251,6 +262,9 @@ pub enum OpenDalStorage { Azdls { /// Azure DLS configuration. config: Arc, + /// Custom AZDLS credential loader. + #[serde(skip)] + customized_credential_load: Option, }, /// HuggingFace Hub storage variant. /// @@ -375,7 +389,10 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-azdls")] - OpenDalStorage::Azdls { config } => azdls_create_operator(path, config)?, + OpenDalStorage::Azdls { + config, + customized_credential_load, + } => azdls_create_operator(path, customized_credential_load, config)?, #[cfg(feature = "opendal-hf")] OpenDalStorage::Hf { config } => hf_config_build(config, path)?, #[cfg(all( @@ -493,7 +510,7 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-azdls")] - OpenDalStorage::Azdls { config } => { + OpenDalStorage::Azdls { config, .. } => { let azure_path = path.parse::()?; match_path_with_config(&azure_path, config)?; let relative_path_len = azure_path.path.len(); @@ -800,6 +817,7 @@ mod tests { endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), ..Default::default() }), + customized_credential_load: None, }; assert_eq!( diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index db47fe0cf8..f22030f4c3 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -92,6 +92,8 @@ struct CustomCredentialLoaders { s3: Option, #[cfg(feature = "opendal-gcs")] gcs: Option, + #[cfg(feature = "opendal-azdls")] + azdls: Option, } /// A resolving storage factory that creates [`OpenDalResolvingStorage`] instances. @@ -145,6 +147,16 @@ impl OpenDalResolvingStorageFactory { self.loaders.gcs = Some(loader); self } + + /// Set a custom AZDLS credential loader for AZDLS storage. + #[cfg(feature = "opendal-azdls")] + pub fn with_azdls_credential_loader( + mut self, + loader: crate::azdls::CustomAzdlsCredentialLoader, + ) -> Self { + self.loaders.azdls = Some(loader); + self + } } #[typetag::serde] @@ -210,6 +222,7 @@ impl OpenDalResolvingStorage { let config = crate::azdls::azdls_config_parse(self.props.clone())?; Ok(OpenDalStorage::Azdls { config: Arc::new(config), + customized_credential_load: self.loaders.azdls.clone(), }) } #[cfg(feature = "opendal-fs")] From 2873176d3e1390d0de4a70dacc8c43ccda5f0aeb Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Thu, 10 Sep 2026 16:58:26 -0400 Subject: [PATCH 3/8] add test for custom cred loader --- crates/storage/opendal/src/azdls.rs | 81 ++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/crates/storage/opendal/src/azdls.rs b/crates/storage/opendal/src/azdls.rs index 55c5c66e6f..d638e7140a 100644 --- a/crates/storage/opendal/src/azdls.rs +++ b/crates/storage/opendal/src/azdls.rs @@ -366,7 +366,10 @@ mod tests { use opendal::services::AzdlsConfig; - use super::{AzureStoragePath, AzureStorageScheme, azdls_config_parse, azdls_create_operator}; + use super::{ + AzdlsCredential, AzureStoragePath, AzureStorageScheme, CustomAzdlsCredentialLoader, + ProvideCredential, azdls_config_parse, azdls_create_operator, + }; #[test] fn test_azdls_config_parse() { @@ -525,6 +528,82 @@ mod tests { } } + #[derive(Debug)] + struct StubProvider; + + impl ProvideCredential for StubProvider { + type Credential = AzdlsCredential; + + async fn provide_credential( + &self, + _ctx: &reqsign_core::Context, + ) -> reqsign_core::Result> { + Ok(None) + } + } + + #[test] + fn test_azdls_create_operator_accepts_credential_loader() { + let config = AzdlsConfig { + account_name: Some("myaccount".to_string()), + endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), + ..Default::default() + }; + let loader = Some(CustomAzdlsCredentialLoader::new(StubProvider)); + + let (op, relative_path) = azdls_create_operator( + "abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet", + &loader, + &config, + ) + .expect("operator builds with a custom credential loader"); + + // Installing a loader must not disturb filesystem or path resolution. + assert_eq!(op.info().name(), "myfs"); + assert_eq!(relative_path, "/path/to/file.parquet"); + } + + #[tokio::test] + async fn test_azdls_credential_loader_replaces_configured_credentials() { + // ADLS replaces the provider chain rather than prepending to it, so a loader that + // yields nothing must fail signing outright rather than falling back to the account key + // below. Signing with that key instead would mean the loader was never installed, and + // the request would get far enough to fail for some other reason. + let config = AzdlsConfig { + account_name: Some("myaccount".to_string()), + account_key: Some("dGVzdGtleQ==".to_string()), + endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), + ..Default::default() + }; + let loader = Some(CustomAzdlsCredentialLoader::new(StubProvider)); + + let (op, relative_path) = azdls_create_operator( + "abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet", + &loader, + &config, + ) + .expect("operator builds with a custom credential loader"); + + let err = op + .exists(relative_path) + .await + .expect_err("a loader that yields nothing must not fall back to the account key"); + assert!( + err.to_string().contains("signing credential"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_custom_azdls_credential_loader_debug_is_redacted() { + let loader = CustomAzdlsCredentialLoader::new(StubProvider); + assert_eq!( + format!("{loader:?}"), + "CustomAzdlsCredentialLoader { .. }", + "the loader's Debug must not reach into the provider it wraps" + ); + } + #[test] fn test_azure_storage_path_parse() { let test_cases = vec![ From 5d1a52a77cb5ccda8ea49b60ddacb20007227bad Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Fri, 11 Sep 2026 11:29:47 -0400 Subject: [PATCH 4/8] fmt cargo file --- crates/storage/opendal/Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index d650bc407c..4849984fea 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -39,7 +39,11 @@ opendal-all = [ "opendal-hf", ] -opendal-azdls = ["opendal/services-azdls", "reqsign-azure-storage", "reqsign-core"] +opendal-azdls = [ + "opendal/services-azdls", + "reqsign-azure-storage", + "reqsign-core", +] opendal-fs = ["opendal/services-fs"] opendal-gcs = ["opendal/services-gcs", "reqsign-google", "reqsign-core"] opendal-hf = ["opendal/services-hf"] From eafda25053000154846871b36ed204e1bac03fb7 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Mon, 14 Sep 2026 10:18:53 -0400 Subject: [PATCH 5/8] fix test_opendal_gcs_chain_falls_through_to_the_config_token --- crates/storage/opendal/tests/file_io_gcs_test.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/storage/opendal/tests/file_io_gcs_test.rs b/crates/storage/opendal/tests/file_io_gcs_test.rs index 42188512fa..65bbf5026e 100644 --- a/crates/storage/opendal/tests/file_io_gcs_test.rs +++ b/crates/storage/opendal/tests/file_io_gcs_test.rs @@ -202,6 +202,11 @@ mod tests { /// succeeds, signed with the credential the loader was supposed to displace. #[tokio::test] async fn test_opendal_gcs_chain_falls_through_to_the_config_token() { + // Every other test here reaches OpenDAL through `FileIO`, which installs the HTTP + // transport on the way past. This one builds its operator directly, so it has to do + // that itself rather than depend on a sibling test having run first. + opendal::install_default(); + let (endpoint, authorization) = serve_one_request().await; // Disabling the ambient sources leaves the chain with exactly two entries that can From 6ed5125a648e68ae009088d3644129f48802e9eb Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Wed, 16 Sep 2026 11:17:55 -0400 Subject: [PATCH 6/8] update PR From f1d5ed4dc518f5a63622c7b5ef67a09636c971a5 Mon Sep 17 00:00:00 2001 From: Sreesh Maheshwar Date: Fri, 11 Sep 2026 23:44:53 +0000 Subject: [PATCH 7/8] ci: pull MinIO images from Quay (#3202) Docker Hub no longer serves the MinIO server and client repositories, preventing integration test containers from starting. Use the corresponding Quay images with the existing release tags. Generated-by: Codex Co-authored-by: Kevin Liu --- dev/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 9d288580df..48421c9049 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -28,7 +28,7 @@ services: # MinIO - S3-compatible storage (shared by all tests) # ============================================================================= minio: - image: minio/minio:RELEASE.2025-05-24T17-08-30Z + image: quay.io/minio/minio:RELEASE.2025-05-24T17-08-30Z environment: - MINIO_ROOT_USER=admin - MINIO_ROOT_PASSWORD=password @@ -56,7 +56,7 @@ services: depends_on: minio: condition: service_healthy - image: minio/mc:RELEASE.2025-05-21T01-59-54Z + image: quay.io/minio/mc:RELEASE.2025-05-21T01-59-54Z environment: - AWS_ACCESS_KEY_ID=admin - AWS_SECRET_ACCESS_KEY=password From c9cc5211c11094fff85141d4f0d8415a38265b08 Mon Sep 17 00:00:00 2001 From: Patrick Butler Date: Wed, 16 Sep 2026 16:00:10 -0400 Subject: [PATCH 8/8] update docstring --- crates/storage/opendal/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 73b14c716f..7998a7b92f 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -98,7 +98,8 @@ cfg_if! { } /// Trait for types that can asynchronously supply credentials to a custom credential -/// loader, such as [`CustomAwsCredentialLoader`] or [`CustomGcsCredentialLoader`]. +/// loader, including [`CustomAwsCredentialLoader`], [`CustomGcsCredentialLoader`], +/// and [`CustomAzdlsCredentialLoader`]. /// /// Downstream implementors must name the trait through this re-export: a loader built /// against a differently-versioned `reqsign-core` will not satisfy the bound.