From 6679a1946c55a2643f54ffea1109768aea86c066 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 29 Aug 2026 17:27:30 +0300 Subject: [PATCH 01/51] feat(storage): Add HDFS support via opendal services-hdfs-native Adds an opendal-hdfs-native cargo feature plus OpenDalStorageFactory::Hdfs and OpenDalStorage::Hdfs variants in iceberg-storage-opendal, using OpenDAL's services-hdfs-native (pure-Rust HDFS RPC, no JNI/libhdfs). The NameNode for a path resolves as: the hdfs.name-node property when set (comma-separated endpoints enable HA failover), otherwise the path authority. hadoop.-prefixed properties are forwarded to the HDFS client configuration, overriding values loaded from $HADOOP_CONF_DIR. Operators are cached per effective NameNode since each holds live RPC connections. Revives and updates PR #2441 (by @jordepic) against the current storage layer and opendal 0.58, where name_node became mandatory and the comma list is the HA mechanism. Closes #2440 Co-authored-by: Jordan Epstein Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 10 + Cargo.lock | 286 ++++++++++++++++- crates/iceberg/public-api.txt | 27 ++ crates/iceberg/src/io/storage/config/hdfs.rs | 110 +++++++ crates/iceberg/src/io/storage/config/mod.rs | 2 + crates/storage/opendal/Cargo.toml | 1 + crates/storage/opendal/README.md | 3 +- crates/storage/opendal/public-api.txt | 4 + crates/storage/opendal/src/hdfs.rs | 289 +++++++++++++++++ crates/storage/opendal/src/lib.rs | 85 +++++ crates/storage/opendal/src/resolving.rs | 37 +++ .../opendal/tests/file_io_hdfs_test.rs | 292 ++++++++++++++++++ crates/test_utils/src/lib.rs | 9 + dev/docker-compose.yaml | 43 +++ dev/hdfs/core-site.xml | 27 ++ dev/hdfs/hdfs-site.xml | 35 +++ 16 files changed, 1247 insertions(+), 13 deletions(-) create mode 100644 crates/iceberg/src/io/storage/config/hdfs.rs create mode 100644 crates/storage/opendal/src/hdfs.rs create mode 100644 crates/storage/opendal/tests/file_io_hdfs_test.rs create mode 100644 dev/hdfs/core-site.xml create mode 100644 dev/hdfs/hdfs-site.xml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 394781eb0b..89a902b904 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,6 +232,16 @@ jobs: cargo test --no-fail-fast ${{ matrix.test-suite.args }} fi + # HDFS tests are `#[ignore]`d (docker fixture needs Linux host + # networking); opt them in here by test-name filter. + - name: Run HDFS integration tests + if: matrix.test-suite.name == 'default' + shell: bash + run: | + cargo nextest run ${{ matrix.test-suite.args }} \ + --run-ignored=only \ + -E 'test(file_io_hdfs)' + - name: Stop Docker containers if: always() && matrix.test-suite.name == 'default' run: make docker-down diff --git a/Cargo.lock b/Cargo.lock index 3802cc03ff..d143485a5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,10 +25,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if 1.0.4", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] +[[package]] +name = "aes" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.0", +] + [[package]] name = "aes-gcm" version = "0.10.3" @@ -36,9 +47,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead", - "aes", - "cipher", - "ctr", + "aes 0.8.4", + "cipher 0.4.4", + "ctr 0.9.2", "ghash", "subtle", ] @@ -1193,6 +1204,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bnum" version = "0.12.1" @@ -1335,7 +1355,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher 0.5.2", ] [[package]] @@ -1410,7 +1439,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.2", ] [[package]] @@ -1642,6 +1682,12 @@ dependencies = [ "futures-io", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1806,7 +1852,16 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher 0.5.2", ] [[package]] @@ -2816,6 +2871,15 @@ dependencies = [ "syn", ] +[[package]] +name = "des" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +dependencies = [ + "cipher 0.5.2", +] + [[package]] name = "diff" version = "0.1.13" @@ -2893,6 +2957,18 @@ dependencies = [ "const-random", ] +[[package]] +name = "dns-lookup" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" +dependencies = [ + "cfg-if 1.0.4", + "libc", + "socket2 0.6.4", + "windows-sys 0.60.2", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -3319,6 +3395,34 @@ dependencies = [ "slab", ] +[[package]] +name = "g2gen" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a7e0eb46f83a20260b850117d204366674e85d3a908d90865c78df9a6b1dfc" +dependencies = [ + "g2poly", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "g2p" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "539e2644c030d3bf4cd208cb842d2ce2f80e82e6e8472390bcef83ceba0d80ad" +dependencies = [ + "g2gen", + "g2poly", +] + +[[package]] +name = "g2poly" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312d2295c7302019c395cfb90dacd00a82a2eabd700429bba9c7a3f38dbbe11b" + [[package]] name = "gearhash" version = "0.1.3" @@ -3500,6 +3604,47 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hdfs-native" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cf4000cab03f95f56fac82175a2978c21517e6072e577ac8d402f00ce996997" +dependencies = [ + "aes 0.9.3", + "base64 0.22.1", + "bitflags 2.13.0", + "bumpalo", + "bytes", + "cbc 0.2.1", + "chrono", + "cipher 0.5.2", + "crc", + "ctr 0.10.1", + "des", + "dns-lookup", + "futures", + "g2p", + "hex", + "hmac 0.13.0", + "libc", + "libloading", + "log", + "md-5 0.11.0", + "num-traits", + "once_cell", + "prost", + "prost-types", + "rand 0.10.2", + "regex", + "roxmltree", + "socket2 0.6.4", + "thiserror 2.0.18", + "tokio", + "url", + "uuid", + "whoami 2.1.2", +] + [[package]] name = "heapify" version = "0.2.0" @@ -4228,10 +4373,20 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", + "block-padding 0.3.3", "generic-array", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding 0.4.2", + "hybrid-array", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -5173,6 +5328,7 @@ dependencies = [ "opendal-service-azdls", "opendal-service-fs", "opendal-service-gcs", + "opendal-service-hdfs-native", "opendal-service-hf", "opendal-service-oss", "opendal-service-s3", @@ -5327,6 +5483,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-hdfs-native" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeb0c6fd99ffe3acf9c58f9664842ea549e4fd984c8202a5e656d18de46dae4" +dependencies = [ + "bytes", + "futures", + "hdfs-native", + "log", + "opendal-core", + "serde", +] + [[package]] name = "opendal-service-hf" version = "0.58.1" @@ -5652,8 +5822,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ - "aes", - "cbc", + "aes 0.8.4", + "cbc 0.1.2", "der", "pbkdf2", "scrypt", @@ -5823,6 +5993,15 @@ dependencies = [ "syn", ] +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + [[package]] name = "psm" version = "0.1.31" @@ -6535,6 +6714,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rsa" version = "0.9.10" @@ -6747,7 +6935,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -8800,6 +8988,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -8833,13 +9030,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows-threading" version = "0.2.1" @@ -8861,6 +9075,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -8873,6 +9093,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -8885,12 +9111,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -8903,6 +9141,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -8915,6 +9159,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -8927,6 +9177,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -8939,6 +9195,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.7.15" diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 4aeddb665a..757fc4abb2 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -780,6 +780,28 @@ impl serde_core::ser::Serialize for iceberg::io::GcsConfig pub fn iceberg::io::GcsConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::GcsConfig pub fn iceberg::io::GcsConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::HdfsConfig +pub iceberg::io::HdfsConfig::name_node: core::option::Option +pub iceberg::io::HdfsConfig::options: std::collections::hash::map::HashMap +impl core::clone::Clone for iceberg::io::HdfsConfig +pub fn iceberg::io::HdfsConfig::clone(&self) -> iceberg::io::HdfsConfig +impl core::cmp::Eq for iceberg::io::HdfsConfig +impl core::cmp::PartialEq for iceberg::io::HdfsConfig +pub fn iceberg::io::HdfsConfig::eq(&self, other: &iceberg::io::HdfsConfig) -> bool +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HdfsConfig +pub type iceberg::io::HdfsConfig::Error = iceberg::Error +pub fn iceberg::io::HdfsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::default::Default for iceberg::io::HdfsConfig +pub fn iceberg::io::HdfsConfig::default() -> iceberg::io::HdfsConfig +impl core::fmt::Debug for iceberg::io::HdfsConfig +pub fn iceberg::io::HdfsConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::HdfsConfig +impl iceberg::io::HdfsConfig +pub fn iceberg::io::HdfsConfig::builder() -> HdfsConfigBuilder<((), ())> +impl serde_core::ser::Serialize for iceberg::io::HdfsConfig +pub fn iceberg::io::HdfsConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::HdfsConfig +pub fn iceberg::io::HdfsConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::io::HfConfig pub iceberg::io::HfConfig::endpoint: core::option::Option pub iceberg::io::HfConfig::revision: core::option::Option @@ -979,6 +1001,9 @@ pub fn iceberg::io::AzdlsConfig::try_from(config: &iceberg::io::StorageConfig) - impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::GcsConfig pub type iceberg::io::GcsConfig::Error = iceberg::Error pub fn iceberg::io::GcsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result +impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HdfsConfig +pub type iceberg::io::HdfsConfig::Error = iceberg::Error +pub fn iceberg::io::HdfsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HfConfig pub type iceberg::io::HfConfig::Error = iceberg::Error pub fn iceberg::io::HfConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result @@ -1015,6 +1040,8 @@ pub const iceberg::io::GCS_PROJECT_ID: &str pub const iceberg::io::GCS_SERVICE_HOST: &str pub const iceberg::io::GCS_TOKEN: &str pub const iceberg::io::GCS_USER_PROJECT: &str +pub const iceberg::io::HDFS_HADOOP_CONF_PREFIX: &str +pub const iceberg::io::HDFS_NAME_NODE: &str pub const iceberg::io::HF_ENDPOINT: &str pub const iceberg::io::HF_REVISION: &str pub const iceberg::io::HF_TOKEN: &str diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs new file mode 100644 index 0000000000..9c0d99734b --- /dev/null +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HDFS storage configuration. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use typed_builder::TypedBuilder; + +use super::StorageConfig; +use crate::Result; + +/// HDFS NameNode RPC endpoint(s), e.g. `hdfs://namenode:8020`; a +/// comma-separated list enables HA failover. When unset, the NameNode is +/// derived from the path authority. +pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; +/// Prefix for properties forwarded to the HDFS client configuration, e.g. +/// `hadoop.dfs.client.failover.random.order`. Forwarded values (prefix +/// stripped) override those loaded from `$HADOOP_CONF_DIR`. +pub const HDFS_HADOOP_CONF_PREFIX: &str = "hadoop."; + +/// HDFS storage configuration. +/// +/// This struct contains all the configuration options for connecting to HDFS. +/// Use the builder pattern via `HdfsConfig::builder()` to construct instances. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] +pub struct HdfsConfig { + /// NameNode endpoint(s); comma-separated for HA failover. + #[builder(default, setter(strip_option, into))] + pub name_node: Option, + /// Extra HDFS client configuration (the `hadoop.` prefix stripped). + #[builder(default)] + pub options: HashMap, +} + +impl TryFrom<&StorageConfig> for HdfsConfig { + type Error = crate::Error; + + fn try_from(config: &StorageConfig) -> Result { + let props = config.props(); + + let mut cfg = HdfsConfig::default(); + + if let Some(name_node) = props.get(HDFS_NAME_NODE) { + cfg.name_node = Some(name_node.clone()); + } + for (key, value) in props { + if let Some(stripped) = key.strip_prefix(HDFS_HADOOP_CONF_PREFIX) { + cfg.options.insert(stripped.to_string(), value.clone()); + } + } + + Ok(cfg) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hdfs_config_builder() { + let cfg = HdfsConfig::builder() + .name_node("hdfs://nn1:8020,hdfs://nn2:8020") + .build(); + assert_eq!( + cfg.name_node.as_deref(), + Some("hdfs://nn1:8020,hdfs://nn2:8020") + ); + assert!(cfg.options.is_empty()); + } + + #[test] + fn test_hdfs_config_from_storage_config() { + let storage_config = StorageConfig::new() + .with_prop(HDFS_NAME_NODE, "hdfs://namenode:8020") + .with_prop("hadoop.dfs.client.failover.random.order", "true") + .with_prop("unrelated.key", "ignored"); + + let cfg = HdfsConfig::try_from(&storage_config).unwrap(); + assert_eq!(cfg.name_node.as_deref(), Some("hdfs://namenode:8020")); + assert_eq!( + cfg.options.get("dfs.client.failover.random.order"), + Some(&"true".to_string()) + ); + assert!(!cfg.options.contains_key("unrelated.key")); + } + + #[test] + fn test_hdfs_config_empty() { + let cfg = HdfsConfig::try_from(&StorageConfig::new()).unwrap(); + assert_eq!(cfg.name_node, None); + assert!(cfg.options.is_empty()); + } +} diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index d8d356de16..fe545d7a5b 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -32,6 +32,7 @@ mod azdls; mod gcs; +mod hdfs; mod hf; mod oss; mod s3; @@ -40,6 +41,7 @@ use std::collections::HashMap; pub use azdls::*; pub use gcs::*; +pub use hdfs::*; pub use hf::*; pub use oss::*; pub use s3::*; diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index e43e7845b3..de1ac7b143 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -42,6 +42,7 @@ opendal-all = [ opendal-azdls = ["opendal/services-azdls"] opendal-fs = ["opendal/services-fs"] opendal-gcs = ["opendal/services-gcs"] +opendal-hdfs-native = ["opendal/services-hdfs-native"] opendal-hf = ["opendal/services-hf"] opendal-memory = ["opendal/services-memory"] opendal-oss = ["opendal/services-oss"] diff --git a/crates/storage/opendal/README.md b/crates/storage/opendal/README.md index 5db4293e67..45fa27e582 100644 --- a/crates/storage/opendal/README.md +++ b/crates/storage/opendal/README.md @@ -32,10 +32,11 @@ OpenDAL-based storage backend implementations for [Apache Iceberg Rust](https:// | Hugging Face | `opendal-hf` | โœ… Stable | Hugging Face buckets and repositories | | Alibaba Cloud OSS | `opendal-oss` | ๐Ÿงช Experimental | Alibaba Cloud Object Storage Service | | Azure Datalake | `opendal-azdls` | ๐Ÿงช Experimental | Azure Datalake Storage v2 | +| HDFS | `opendal-hdfs-native` | ๐Ÿงช Experimental | HDFS via the native Rust client (`hdfs-native`) | You can enable all stable storage backends at once using the `opendal-all` feature flag. -> Note that `opendal-oss` and `opendal-azdls` are currently experimental and not included in `opendal-all`. +> Note that `opendal-oss`, `opendal-azdls` and `opendal-hdfs-native` are currently experimental and not included in `opendal-all`. ## Usage diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index d8c4ecdb38..2a562eaaf3 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -6,6 +6,9 @@ pub iceberg_storage_opendal::OpenDalStorage::Azdls pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::Gcs pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::Hdfs +pub iceberg_storage_opendal::OpenDalStorage::Hdfs::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::Hdfs::operators: alloc::sync::Arc>> pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::LocalFs @@ -39,6 +42,7 @@ pub enum iceberg_storage_opendal::OpenDalStorageFactory pub iceberg_storage_opendal::OpenDalStorageFactory::Azdls pub iceberg_storage_opendal::OpenDalStorageFactory::Fs pub iceberg_storage_opendal::OpenDalStorageFactory::Gcs +pub iceberg_storage_opendal::OpenDalStorageFactory::Hdfs pub iceberg_storage_opendal::OpenDalStorageFactory::Hf pub iceberg_storage_opendal::OpenDalStorageFactory::Memory pub iceberg_storage_opendal::OpenDalStorageFactory::Oss diff --git a/crates/storage/opendal/src/hdfs.rs b/crates/storage/opendal/src/hdfs.rs new file mode 100644 index 0000000000..1c0dfd3da4 --- /dev/null +++ b/crates/storage/opendal/src/hdfs.rs @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). + +use std::collections::HashMap; +use std::sync::RwLock; + +use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_NAME_NODE}; +use iceberg::{Error, ErrorKind, Result}; +use opendal::Operator; +use opendal::services::HdfsNativeConfig; +use url::Url; + +use crate::utils::from_opendal_error; + +/// Parse iceberg properties to [`HdfsNativeConfig`]. +pub(crate) fn hdfs_config_parse(mut m: HashMap) -> Result { + let mut cfg = HdfsNativeConfig::default(); + + if let Some(name_node) = m.remove(HDFS_NAME_NODE) { + cfg.name_node = Some(name_node); + } + + let options: HashMap = m + .into_iter() + .filter_map(|(key, value)| { + key.strip_prefix(HDFS_HADOOP_CONF_PREFIX) + .map(|stripped| (stripped.to_string(), value)) + }) + .collect(); + if !options.is_empty() { + cfg.options = Some(options); + } + + Ok(cfg) +} + +/// Parse an HDFS path into `Some("hdfs://")` (`None` when +/// authority-less) and the relative path (no leading `/`, opendal style). +pub(crate) fn parse_hdfs_path(path: &str) -> Result<(Option, &str)> { + let url = Url::parse(path).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path: {path}: {e}"), + ) + })?; + if url.scheme() != "hdfs" { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path: {path}, expected scheme `hdfs://`"), + )); + } + + let name_node = url.host_str().filter(|h| !h.is_empty()).map(|host| { + url.port() + .map(|port| format!("hdfs://{host}:{port}")) + .unwrap_or_else(|| format!("hdfs://{host}")) + }); + + // `url.path()` borrows from `url` and can't be returned with the input's + // lifetime. Slice the path component out of the original input instead; + // it starts after the first `/` following the `hdfs://` prefix. Opendal + // paths must not start with `/` (`Deleter::delete` rejects them). + let after_scheme = &path["hdfs://".len()..]; + let rel = match after_scheme.find('/') { + Some(i) => after_scheme[i..].trim_start_matches('/'), + None => "", + }; + + Ok((name_node, rel)) +} + +/// Creates an operator for the path, cached per effective NameNode (the +/// configured `hdfs.name-node`, else the path authority) โ€” each operator +/// holds an HDFS client with live RPC connections. +pub(crate) fn hdfs_create_operator<'a>( + path: &'a str, + config: &HdfsNativeConfig, + operators: &RwLock>, +) -> Result<(Operator, &'a str)> { + let (authority_name_node, relative_path) = parse_hdfs_path(path)?; + + let name_node = match config.name_node.clone().or(authority_name_node) { + Some(name_node) => name_node, + None => { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid hdfs path: {path}, authority-less paths require the `{HDFS_NAME_NODE}` property" + ), + )); + } + }; + + // Fast path: check read lock first. + { + let cache = operators + .read() + .map_err(|_| Error::new(ErrorKind::Unexpected, "HDFS operator cache lock poisoned"))?; + if let Some(op) = cache.get(&name_node) { + return Ok((op.clone(), relative_path)); + } + } + + // Slow path: build and insert under write lock, re-checking for a + // concurrent insert. + let mut cache = operators + .write() + .map_err(|_| Error::new(ErrorKind::Unexpected, "HDFS operator cache lock poisoned"))?; + let op = match cache.get(&name_node) { + Some(op) => op.clone(), + None => { + let op = hdfs_operator_build(config, &name_node)?; + cache.insert(name_node, op.clone()); + op + } + }; + + Ok((op, relative_path)) +} + +/// Build a new OpenDAL [`Operator`]: OpenDAL splits `name_node` on commas +/// into a synthetic HA name service; `$HADOOP_CONF_DIR` XML still merges in. +fn hdfs_operator_build(config: &HdfsNativeConfig, name_node: &str) -> Result { + let mut cfg = config.clone(); + cfg.name_node = Some(name_node.to_string()); + Operator::from_config(cfg).map_err(from_opendal_error) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hdfs_config_parse_name_node_and_options() { + let props = HashMap::from([ + ( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), + ), + ( + "hadoop.dfs.client.failover.random.order".to_string(), + "true".to_string(), + ), + ("unrelated.key".to_string(), "ignored".to_string()), + ]); + + let cfg = hdfs_config_parse(props).unwrap(); + + assert_eq!( + cfg.name_node.as_deref(), + Some("hdfs://nn1:8020,hdfs://nn2:8020") + ); + let options = cfg.options.unwrap(); + assert_eq!( + options.get("dfs.client.failover.random.order"), + Some(&"true".to_string()) + ); + assert!(!options.contains_key("unrelated.key")); + } + + #[test] + fn test_hdfs_config_parse_empty() { + let cfg = hdfs_config_parse(HashMap::new()).unwrap(); + + assert_eq!(cfg.name_node, None); + assert_eq!(cfg.options, None); + } + + #[test] + fn test_parse_hdfs_path_with_authority_and_rel() { + let (nn, rel) = parse_hdfs_path("hdfs://nameservice1/a/b").unwrap(); + + assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); + assert_eq!(rel, "a/b"); + } + + #[test] + fn test_parse_hdfs_path_with_authority_and_port() { + let (nn, rel) = parse_hdfs_path("hdfs://nn:8020/foo").unwrap(); + + assert_eq!(nn.as_deref(), Some("hdfs://nn:8020")); + assert_eq!(rel, "foo"); + } + + #[test] + fn test_parse_hdfs_path_with_authority_no_path() { + let (nn, rel) = parse_hdfs_path("hdfs://nameservice1").unwrap(); + + assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); + assert_eq!(rel, ""); + } + + #[test] + fn test_parse_hdfs_path_with_authority_trailing_slash() { + let (nn, rel) = parse_hdfs_path("hdfs://nameservice1/").unwrap(); + + assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); + assert_eq!(rel, ""); + } + + #[test] + fn test_parse_hdfs_path_authority_less_returns_none() { + let (nn, rel) = parse_hdfs_path("hdfs:///a/b").unwrap(); + + assert_eq!(nn, None); + assert_eq!(rel, "a/b"); + } + + #[test] + fn test_parse_hdfs_path_wrong_scheme_errors() { + let err = parse_hdfs_path("file:///tmp/x").unwrap_err(); + + assert!(err.to_string().contains("expected scheme `hdfs://`")); + } + + #[test] + fn test_parse_hdfs_path_invalid_url_errors() { + let err = parse_hdfs_path("not-a-url").unwrap_err(); + + assert!(err.to_string().contains("Invalid hdfs path")); + } + + #[test] + fn test_hdfs_create_operator_configured_name_node_wins() { + let config = hdfs_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + "hdfs://configured:8020".to_string(), + )])) + .unwrap(); + let operators = RwLock::new(HashMap::new()); + + let (_, rel) = + hdfs_create_operator("hdfs://from-path:9000/a/b", &config, &operators).unwrap(); + + assert_eq!(rel, "a/b"); + let cache = operators.read().unwrap(); + assert!(cache.contains_key("hdfs://configured:8020")); + assert!(!cache.contains_key("hdfs://from-path:9000")); + } + + #[test] + fn test_hdfs_create_operator_uses_path_authority() { + let config = HdfsNativeConfig::default(); + let operators = RwLock::new(HashMap::new()); + + let (_, rel) = hdfs_create_operator("hdfs://nn:8020/a/b", &config, &operators).unwrap(); + + assert_eq!(rel, "a/b"); + assert!(operators.read().unwrap().contains_key("hdfs://nn:8020")); + } + + #[test] + fn test_hdfs_create_operator_caches_per_name_node() { + let config = HdfsNativeConfig::default(); + let operators = RwLock::new(HashMap::new()); + + hdfs_create_operator("hdfs://nn1:8020/a", &config, &operators).unwrap(); + hdfs_create_operator("hdfs://nn1:8020/b", &config, &operators).unwrap(); + hdfs_create_operator("hdfs://nn2:8020/c", &config, &operators).unwrap(); + + assert_eq!(operators.read().unwrap().len(), 2); + } + + #[test] + fn test_hdfs_create_operator_authority_less_without_config_errors() { + let config = HdfsNativeConfig::default(); + let operators = RwLock::new(HashMap::new()); + + let err = hdfs_create_operator("hdfs:///a/b", &config, &operators).unwrap_err(); + + assert!(err.to_string().contains(HDFS_NAME_NODE)); + } +} diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index d02507e7b5..6626c356e1 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -74,6 +74,16 @@ cfg_if! { } } +cfg_if! { + if #[cfg(feature = "opendal-hdfs-native")] { + mod hdfs; + use std::sync::RwLock; + + use hdfs::*; + use opendal::services::HdfsNativeConfig; + } +} + cfg_if! { if #[cfg(feature = "opendal-memory")] { mod memory; @@ -122,6 +132,9 @@ pub enum OpenDalStorageFactory { /// GCS storage factory. #[cfg(feature = "opendal-gcs")] Gcs, + /// HDFS storage factory. + #[cfg(feature = "opendal-hdfs-native")] + Hdfs, /// OSS storage factory. #[cfg(feature = "opendal-oss")] Oss, @@ -155,6 +168,11 @@ impl StorageFactory for OpenDalStorageFactory { OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs { config: gcs_config_parse(config.props().clone())?.into(), })), + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorageFactory::Hdfs => Ok(Arc::new(OpenDalStorage::Hdfs { + config: hdfs_config_parse(config.props().clone())?.into(), + operators: Arc::new(RwLock::new(HashMap::new())), + })), #[cfg(feature = "opendal-oss")] OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss { config: oss_config_parse(config.props().clone())?.into(), @@ -175,6 +193,7 @@ impl StorageFactory for OpenDalStorageFactory { not(feature = "opendal-oss"), not(feature = "opendal-azdls"), not(feature = "opendal-hf"), + not(feature = "opendal-hdfs-native"), ))] _ => Err(Error::new( ErrorKind::FeatureUnsupported, @@ -217,6 +236,18 @@ pub enum OpenDalStorage { /// GCS configuration. config: Arc, }, + /// HDFS storage variant. + /// + /// The NameNode is taken from the `hdfs.name-node` property when set + /// (comma-separated endpoints enable HA failover), else the path authority. + #[cfg(feature = "opendal-hdfs-native")] + Hdfs { + /// HDFS configuration. + config: Arc, + /// Operator cache keyed by effective NameNode. + #[serde(skip, default)] + operators: Arc>>, + }, /// OSS storage variant. #[cfg(feature = "opendal-oss")] Oss { @@ -322,6 +353,10 @@ impl OpenDalStorage { )); } } + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorage::Hdfs { config, operators } => { + hdfs_create_operator(path, config, operators)? + } #[cfg(feature = "opendal-oss")] OpenDalStorage::Oss { config } => { let op = oss_config_build(config, path)?; @@ -346,6 +381,7 @@ impl OpenDalStorage { not(feature = "opendal-oss"), not(feature = "opendal-azdls"), not(feature = "opendal-hf"), + not(feature = "opendal-hdfs-native"), ))] _ => { return Err(Error::new( @@ -434,6 +470,11 @@ impl OpenDalStorage { )) } } + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorage::Hdfs { .. } => { + let (_, relative_path) = parse_hdfs_path(path)?; + Ok(relative_path) + } #[cfg(feature = "opendal-oss")] OpenDalStorage::Oss { .. } => { let url = url::Url::parse(path)?; @@ -474,6 +515,7 @@ impl OpenDalStorage { not(feature = "opendal-oss"), not(feature = "opendal-azdls"), not(feature = "opendal-hf"), + not(feature = "opendal-hdfs-native"), ))] _ => Err(Error::new( ErrorKind::FeatureUnsupported, @@ -750,6 +792,49 @@ mod tests { ); } + #[cfg(feature = "opendal-hdfs-native")] + fn hdfs_test_storage() -> OpenDalStorage { + OpenDalStorage::Hdfs { + config: Arc::new(HdfsNativeConfig::default()), + operators: Arc::new(RwLock::new(HashMap::new())), + } + } + + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_relativize_path_hdfs() { + let storage = hdfs_test_storage(); + + assert_eq!( + storage + .relativize_path("hdfs://nameservice1/a/b.parquet") + .unwrap(), + "a/b.parquet" + ); + assert_eq!( + storage + .relativize_path("hdfs://nn:8020/warehouse/db/t") + .unwrap(), + "warehouse/db/t" + ); + } + + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_relativize_path_hdfs_authority_less() { + let storage = hdfs_test_storage(); + + assert_eq!(storage.relativize_path("hdfs:///a/b").unwrap(), "a/b"); + } + + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_relativize_path_hdfs_wrong_scheme_errors() { + let storage = hdfs_test_storage(); + + assert!(storage.relativize_path("s3://bucket/x").is_err()); + } + #[cfg(feature = "opendal-azdls")] #[test] fn test_relativize_path_azdls() { diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 86993220a8..8f7ad22288 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -51,6 +51,7 @@ pub const SCHEME_ABFS: &str = "abfs"; pub const SCHEME_WASBS: &str = "wasbs"; pub const SCHEME_WASB: &str = "wasb"; pub const SCHEME_HF: &str = "hf"; +pub const SCHEME_HDFS: &str = "hdfs"; /// Parse a URL scheme string. fn parse_scheme(scheme: &str) -> Result<&'static str> { @@ -62,6 +63,7 @@ fn parse_scheme(scheme: &str) -> Result<&'static str> { SCHEME_OSS => Ok("oss"), SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok("azdls"), SCHEME_HF => Ok("hf"), + SCHEME_HDFS => Ok("hdfs"), s => Err(Error::new( ErrorKind::FeatureUnsupported, format!("Unsupported storage scheme: {s}"), @@ -127,6 +129,14 @@ fn build_storage_for_scheme( config: Arc::new(config), }) } + #[cfg(feature = "opendal-hdfs-native")] + "hdfs" => { + let config = crate::hdfs::hdfs_config_parse(props.clone())?; + Ok(OpenDalStorage::Hdfs { + config: Arc::new(config), + operators: Arc::new(RwLock::new(HashMap::new())), + }) + } unsupported => Err(Error::new( ErrorKind::FeatureUnsupported, format!("Unsupported storage scheme: {unsupported}"), @@ -353,6 +363,33 @@ mod tests { assert!(Arc::ptr_eq(&a, &c), "s3 and s3n should share one instance"); } + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_resolve_hdfs_returns_hdfs_variant() { + let storage = empty_resolving_storage(); + + let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap(); + + assert!( + matches!(&*resolved, OpenDalStorage::Hdfs { .. }), + "expected Hdfs variant, got {resolved:?}" + ); + } + + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_resolve_hdfs_distinct_authorities_share_instance() { + let storage = empty_resolving_storage(); + + let a = storage.resolve("hdfs://ns1/a").unwrap(); + let b = storage.resolve("hdfs://ns2/b").unwrap(); + + assert!( + Arc::ptr_eq(&a, &b), + "different authorities should share the OpenDalStorage::Hdfs instance (operator cache is internal)" + ); + } + #[cfg(feature = "opendal-azdls")] #[test] fn test_resolve_azdls_aliases_share_instance() { diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs new file mode 100644 index 0000000000..0f7bd00584 --- /dev/null +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for HDFS FileIO via OpenDAL `services-hdfs-native`. +//! +//! These tests need the `hdfs-namenode`/`hdfs-datanode` services from +//! `dev/docker-compose.yaml` and are `#[ignore]`d (host networking is +//! Linux-only); CI opts in via `cargo nextest run --run-ignored=only`. + +#[cfg(feature = "opendal-hdfs-native")] +mod tests { + use std::sync::Arc; + + use bytes::Bytes; + use futures::StreamExt; + use iceberg::io::{FileIO, FileIOBuilder, HDFS_NAME_NODE}; + use iceberg_storage_opendal::{OpenDalResolvingStorageFactory, OpenDalStorageFactory}; + use iceberg_test_utils::{get_hdfs_endpoint, normalize_test_name_with_parts, set_up}; + + fn get_file_io() -> FileIO { + set_up(); + FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Hdfs)).build() + } + + fn test_path(suffix: &str) -> String { + format!( + "{}/{}", + get_hdfs_endpoint(), + normalize_test_name_with_parts!(suffix) + ) + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_exists() { + let file_io = get_file_io(); + + let absent = test_path("test_file_io_hdfs_exists_absent"); + assert!(!file_io.exists(&absent).await.unwrap()); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_write_and_read() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_write_and_read"); + let _ = file_io.delete(&path).await; + + let output = file_io.new_output(&path).unwrap(); + output + .write(Bytes::from_static(b"hello hdfs")) + .await + .unwrap(); + + assert!(file_io.exists(&path).await.unwrap()); + let input = file_io.new_input(&path).unwrap(); + assert_eq!( + input.read().await.unwrap(), + Bytes::from_static(b"hello hdfs") + ); + } + + /// The HA flow: table locations carry a logical authority while + /// `hdfs.name-node` carries the (comma-separated) endpoints; it wins. + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_configured_name_node() { + set_up(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Hdfs)) + .with_prop(HDFS_NAME_NODE, get_hdfs_endpoint()) + .build(); + + // The path authority is a logical name; the configured NameNode wins. + let path = format!( + "hdfs://logical-nameservice/{}", + normalize_test_name_with_parts!("test_file_io_hdfs_configured_name_node") + ); + let _ = file_io.delete(&path).await; + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"via configured name node")) + .await + .unwrap(); + + assert!(file_io.exists(&path).await.unwrap()); + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"via configured name node") + ); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_overwrite() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_overwrite"); + let _ = file_io.delete(&path).await; + + for content in [b"first".as_slice(), b"second, longer".as_slice()] { + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(content)) + .await + .unwrap(); + } + + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"second, longer") + ); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_delete_stream() { + let file_io = get_file_io(); + + let paths: Vec = (0..5) + .map(|i| format!("{}/file-{i}", test_path("test_file_io_hdfs_delete_stream"))) + .collect(); + for path in &paths { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write("delete-me".into()) + .await + .unwrap(); + assert!(file_io.exists(path).await.unwrap()); + } + + let stream = futures::stream::iter(paths.clone()).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + for path in &paths { + assert!(!file_io.exists(path).await.unwrap()); + } + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_delete_stream_empty() { + let file_io = get_file_io(); + let stream = futures::stream::empty().boxed(); + file_io.delete_stream(stream).await.unwrap(); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_resolving_storage() { + set_up(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())).build(); + let path = test_path("test_file_io_hdfs_resolving_storage"); + let _ = file_io.delete(&path).await; + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"resolving")) + .await + .unwrap(); + + assert_eq!( + file_io.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"resolving") + ); + + file_io.delete(&path).await.unwrap(); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_metadata() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_metadata"); + let _ = file_io.delete(&path).await; + let content = Bytes::from_static(b"0123456789"); + + file_io + .new_output(&path) + .unwrap() + .write(content.clone()) + .await + .unwrap(); + + let metadata = file_io.new_input(&path).unwrap().metadata().await.unwrap(); + assert_eq!(metadata.size, content.len() as u64); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_delete() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_delete"); + + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"x")) + .await + .unwrap(); + assert!(file_io.exists(&path).await.unwrap()); + + file_io.delete(&path).await.unwrap(); + assert!(!file_io.exists(&path).await.unwrap()); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_delete_prefix() { + let file_io = get_file_io(); + let dir = test_path("test_file_io_hdfs_delete_prefix"); + let _ = file_io.delete_prefix(&dir).await; + + for i in 0..3 { + let path = format!("{dir}/file_{i}"); + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from(format!("payload {i}"))) + .await + .unwrap(); + } + assert!(file_io.exists(&format!("{dir}/file_0")).await.unwrap()); + + file_io.delete_prefix(&dir).await.unwrap(); + + for i in 0..3 { + assert!(!file_io.exists(&format!("{dir}/file_{i}")).await.unwrap()); + } + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_reader_range() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_reader_range"); + let _ = file_io.delete(&path).await; + let content = Bytes::from_static(b"abcdefghij"); + + file_io + .new_output(&path) + .unwrap() + .write(content.clone()) + .await + .unwrap(); + + let reader = file_io.new_input(&path).unwrap().reader().await.unwrap(); + assert_eq!( + reader.read(0..5).await.unwrap(), + Bytes::from_static(b"abcde") + ); + assert_eq!( + reader.read(5..10).await.unwrap(), + Bytes::from_static(b"fghij") + ); + } + + #[tokio::test] + #[ignore = "Linux-only: HDFS docker fixture uses host networking"] + async fn test_file_io_hdfs_streaming_writer() { + let file_io = get_file_io(); + let path = test_path("test_file_io_hdfs_streaming_writer"); + let _ = file_io.delete(&path).await; + + let output = file_io.new_output(&path).unwrap(); + let mut writer = output.writer().await.unwrap(); + writer.write(Bytes::from_static(b"part1 ")).await.unwrap(); + writer.write(Bytes::from_static(b"part2")).await.unwrap(); + writer.close().await.unwrap(); + + let read = file_io.new_input(&path).unwrap().read().await.unwrap(); + assert_eq!(read, Bytes::from_static(b"part1 part2")); + } +} diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index e44d96c385..f027df12e9 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -42,6 +42,7 @@ mod common { pub const ENV_HMS_ENDPOINT: &str = "ICEBERG_TEST_HMS_ENDPOINT"; pub const ENV_GLUE_ENDPOINT: &str = "ICEBERG_TEST_GLUE_ENDPOINT"; pub const ENV_GCS_ENDPOINT: &str = "ICEBERG_TEST_GCS_ENDPOINT"; + pub const ENV_HDFS_ENDPOINT: &str = "ICEBERG_TEST_HDFS_ENDPOINT"; // Default ports matching dev/docker-compose.yaml pub const DEFAULT_MINIO_PORT: u16 = 9000; @@ -49,6 +50,7 @@ mod common { pub const DEFAULT_HMS_PORT: u16 = 9083; pub const DEFAULT_GLUE_PORT: u16 = 5001; pub const DEFAULT_GCS_PORT: u16 = 4443; + pub const DEFAULT_HDFS_NN_PORT: u16 = 8020; /// Returns the MinIO S3-compatible endpoint. /// Checks ICEBERG_TEST_MINIO_ENDPOINT env var, otherwise returns localhost default. @@ -84,6 +86,13 @@ mod common { .unwrap_or_else(|_| format!("http://localhost:{DEFAULT_GCS_PORT}")) } + /// Returns the HDFS NameNode endpoint (e.g. `hdfs://localhost:8020`). + /// Checks ICEBERG_TEST_HDFS_ENDPOINT env var, otherwise returns localhost default. + pub fn get_hdfs_endpoint() -> String { + std::env::var(ENV_HDFS_ENDPOINT) + .unwrap_or_else(|_| format!("hdfs://localhost:{DEFAULT_HDFS_NN_PORT}")) + } + /// Helper to clean up a namespace and its tables before a test runs. /// This handles the case where previous test runs left data in the persistent database. pub async fn cleanup_namespace(catalog: &C, ns: &NamespaceIdent) { diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 9d288580df..6416e8983f 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -147,6 +147,49 @@ services: timeout: 5s retries: 5 + # ============================================================================= + # HDFS - single-node NameNode + DataNode for HDFS tests + # ============================================================================= + # hdfs-native connects to DataNodes by their registered IP โ€” unroutable on + # a docker bridge, hence host networking (Linux-only; tests are `#[ignore]`d). + hdfs-namenode: + image: apache/hadoop:3.5.0 + network_mode: "host" + command: ["hdfs", "namenode"] + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + extra_hosts: + - "docker-desktop:127.0.0.1" + volumes: + - ./hdfs/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro + - ./hdfs/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro + healthcheck: + test: ["CMD-SHELL", "hdfs dfsadmin -safemode get | grep -q OFF"] + interval: 5s + timeout: 15s + retries: 30 + start_period: 30s + + hdfs-datanode: + image: apache/hadoop:3.5.0 + network_mode: "host" + command: ["hdfs", "datanode"] + depends_on: + hdfs-namenode: + condition: service_healthy + extra_hosts: + - "docker-desktop:127.0.0.1" + volumes: + - ./hdfs/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro + - ./hdfs/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro + # Healthy only once registered with the NameNode โ€” writes fail until then. + healthcheck: + test: ["CMD-SHELL", "hdfs dfsadmin -report | grep -q 'Live datanodes (1)'"] + interval: 5s + timeout: 15s + retries: 30 + start_period: 10s + # ============================================================================= # Fake GCS Server - GCS emulator for GCS tests # ============================================================================= diff --git a/dev/hdfs/core-site.xml b/dev/hdfs/core-site.xml new file mode 100644 index 0000000000..3c57323d7b --- /dev/null +++ b/dev/hdfs/core-site.xml @@ -0,0 +1,27 @@ + + + + + fs.defaultFS + hdfs://localhost:8020 + + + hadoop.http.staticuser.user + root + + diff --git a/dev/hdfs/hdfs-site.xml b/dev/hdfs/hdfs-site.xml new file mode 100644 index 0000000000..5eb2d198f1 --- /dev/null +++ b/dev/hdfs/hdfs-site.xml @@ -0,0 +1,35 @@ + + + + + dfs.replication + 1 + + + dfs.permissions.enabled + false + + + dfs.namenode.name.dir + file:///tmp/hadoop-root/dfs/name + + + dfs.datanode.data.dir + file:///tmp/hadoop-root/dfs/data + + From 2b0ca954e65a57be5f4969c5442ed7557c2f6008 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Mon, 31 Aug 2026 12:17:15 +0300 Subject: [PATCH 02/51] =?UTF-8?q?refactor:=20address=20review=20=E2=80=94?= =?UTF-8?q?=20Properties=20derive=20for=20HdfsConfig,=20drop=20HDFS=20CI?= =?UTF-8?q?=20special-casing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HdfsConfig is now pub(crate) and parsed via #[derive(Properties)] (key/prefix attributes) instead of hand-written TryFrom + TypedBuilder. - HDFS integration tests are no longer #[ignore]d and the dedicated CI step is gone; they run under the default nextest invocation since make docker-up already starts the fixture and the Tests job is Linux-only. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 10 --- crates/iceberg/public-api.txt | 25 ------ crates/iceberg/src/io/storage/config/hdfs.rs | 90 +++++++------------ .../opendal/tests/file_io_hdfs_test.rs | 16 +--- dev/docker-compose.yaml | 2 +- 5 files changed, 33 insertions(+), 110 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89a902b904..394781eb0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,16 +232,6 @@ jobs: cargo test --no-fail-fast ${{ matrix.test-suite.args }} fi - # HDFS tests are `#[ignore]`d (docker fixture needs Linux host - # networking); opt them in here by test-name filter. - - name: Run HDFS integration tests - if: matrix.test-suite.name == 'default' - shell: bash - run: | - cargo nextest run ${{ matrix.test-suite.args }} \ - --run-ignored=only \ - -E 'test(file_io_hdfs)' - - name: Stop Docker containers if: always() && matrix.test-suite.name == 'default' run: make docker-down diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 757fc4abb2..97673c15fb 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -780,28 +780,6 @@ impl serde_core::ser::Serialize for iceberg::io::GcsConfig pub fn iceberg::io::GcsConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::GcsConfig pub fn iceberg::io::GcsConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> -pub struct iceberg::io::HdfsConfig -pub iceberg::io::HdfsConfig::name_node: core::option::Option -pub iceberg::io::HdfsConfig::options: std::collections::hash::map::HashMap -impl core::clone::Clone for iceberg::io::HdfsConfig -pub fn iceberg::io::HdfsConfig::clone(&self) -> iceberg::io::HdfsConfig -impl core::cmp::Eq for iceberg::io::HdfsConfig -impl core::cmp::PartialEq for iceberg::io::HdfsConfig -pub fn iceberg::io::HdfsConfig::eq(&self, other: &iceberg::io::HdfsConfig) -> bool -impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HdfsConfig -pub type iceberg::io::HdfsConfig::Error = iceberg::Error -pub fn iceberg::io::HdfsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result -impl core::default::Default for iceberg::io::HdfsConfig -pub fn iceberg::io::HdfsConfig::default() -> iceberg::io::HdfsConfig -impl core::fmt::Debug for iceberg::io::HdfsConfig -pub fn iceberg::io::HdfsConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::marker::StructuralPartialEq for iceberg::io::HdfsConfig -impl iceberg::io::HdfsConfig -pub fn iceberg::io::HdfsConfig::builder() -> HdfsConfigBuilder<((), ())> -impl serde_core::ser::Serialize for iceberg::io::HdfsConfig -pub fn iceberg::io::HdfsConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::HdfsConfig -pub fn iceberg::io::HdfsConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg::io::HfConfig pub iceberg::io::HfConfig::endpoint: core::option::Option pub iceberg::io::HfConfig::revision: core::option::Option @@ -1001,9 +979,6 @@ pub fn iceberg::io::AzdlsConfig::try_from(config: &iceberg::io::StorageConfig) - impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::GcsConfig pub type iceberg::io::GcsConfig::Error = iceberg::Error pub fn iceberg::io::GcsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result -impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HdfsConfig -pub type iceberg::io::HdfsConfig::Error = iceberg::Error -pub fn iceberg::io::HdfsConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result impl core::convert::TryFrom<&iceberg::io::StorageConfig> for iceberg::io::HfConfig pub type iceberg::io::HfConfig::Error = iceberg::Error pub fn iceberg::io::HfConfig::try_from(config: &iceberg::io::StorageConfig) -> iceberg::Result diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index 9c0d99734b..e2d468dec1 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -19,11 +19,7 @@ use std::collections::HashMap; -use serde::{Deserialize, Serialize}; -use typed_builder::TypedBuilder; - -use super::StorageConfig; -use crate::Result; +use iceberg_property_macro::Properties; /// HDFS NameNode RPC endpoint(s), e.g. `hdfs://namenode:8020`; a /// comma-separated list enables HA failover. When unset, the NameNode is @@ -35,38 +31,17 @@ pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; pub const HDFS_HADOOP_CONF_PREFIX: &str = "hadoop."; /// HDFS storage configuration. -/// -/// This struct contains all the configuration options for connecting to HDFS. -/// Use the builder pattern via `HdfsConfig::builder()` to construct instances. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, TypedBuilder)] -pub struct HdfsConfig { +// No in-crate consumer yet: `iceberg-storage-opendal` parses the raw +// properties itself and only shares the key constants above. +#[allow(dead_code)] +#[derive(Debug, Properties)] +pub(crate) struct HdfsConfig { /// NameNode endpoint(s); comma-separated for HA failover. - #[builder(default, setter(strip_option, into))] - pub name_node: Option, + #[property(key = HDFS_NAME_NODE, default = None, getter)] + name_node: Option, /// Extra HDFS client configuration (the `hadoop.` prefix stripped). - #[builder(default)] - pub options: HashMap, -} - -impl TryFrom<&StorageConfig> for HdfsConfig { - type Error = crate::Error; - - fn try_from(config: &StorageConfig) -> Result { - let props = config.props(); - - let mut cfg = HdfsConfig::default(); - - if let Some(name_node) = props.get(HDFS_NAME_NODE) { - cfg.name_node = Some(name_node.clone()); - } - for (key, value) in props { - if let Some(stripped) = key.strip_prefix(HDFS_HADOOP_CONF_PREFIX) { - cfg.options.insert(stripped.to_string(), value.clone()); - } - } - - Ok(cfg) - } + #[property(prefix = HDFS_HADOOP_CONF_PREFIX, getter)] + options: HashMap, } #[cfg(test)] @@ -74,37 +49,32 @@ mod tests { use super::*; #[test] - fn test_hdfs_config_builder() { - let cfg = HdfsConfig::builder() - .name_node("hdfs://nn1:8020,hdfs://nn2:8020") - .build(); - assert_eq!( - cfg.name_node.as_deref(), - Some("hdfs://nn1:8020,hdfs://nn2:8020") - ); - assert!(cfg.options.is_empty()); - } - - #[test] - fn test_hdfs_config_from_storage_config() { - let storage_config = StorageConfig::new() - .with_prop(HDFS_NAME_NODE, "hdfs://namenode:8020") - .with_prop("hadoop.dfs.client.failover.random.order", "true") - .with_prop("unrelated.key", "ignored"); - - let cfg = HdfsConfig::try_from(&storage_config).unwrap(); - assert_eq!(cfg.name_node.as_deref(), Some("hdfs://namenode:8020")); + fn test_hdfs_config_from_properties() { + let props = HashMap::from([ + ( + HDFS_NAME_NODE.to_string(), + "hdfs://namenode:8020".to_string(), + ), + ( + "hadoop.dfs.client.failover.random.order".to_string(), + "true".to_string(), + ), + ("unrelated.key".to_string(), "ignored".to_string()), + ]); + + let cfg = HdfsConfig::from_properties(&props).unwrap(); + assert_eq!(cfg.name_node().as_deref(), Some("hdfs://namenode:8020")); assert_eq!( - cfg.options.get("dfs.client.failover.random.order"), + cfg.options().get("dfs.client.failover.random.order"), Some(&"true".to_string()) ); - assert!(!cfg.options.contains_key("unrelated.key")); + assert!(!cfg.options().contains_key("unrelated.key")); } #[test] fn test_hdfs_config_empty() { - let cfg = HdfsConfig::try_from(&StorageConfig::new()).unwrap(); - assert_eq!(cfg.name_node, None); - assert!(cfg.options.is_empty()); + let cfg = HdfsConfig::from_properties(&HashMap::new()).unwrap(); + assert_eq!(cfg.name_node().as_deref(), None); + assert!(cfg.options().is_empty()); } } diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index 0f7bd00584..f072a9c476 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -18,8 +18,8 @@ //! Integration tests for HDFS FileIO via OpenDAL `services-hdfs-native`. //! //! These tests need the `hdfs-namenode`/`hdfs-datanode` services from -//! `dev/docker-compose.yaml` and are `#[ignore]`d (host networking is -//! Linux-only); CI opts in via `cargo nextest run --run-ignored=only`. +//! `dev/docker-compose.yaml` (started by `make docker-up`); the fixture +//! uses host networking, which needs Linux or a recent Docker runtime. #[cfg(feature = "opendal-hdfs-native")] mod tests { @@ -45,7 +45,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_exists() { let file_io = get_file_io(); @@ -54,7 +53,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_write_and_read() { let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_write_and_read"); @@ -77,7 +75,6 @@ mod tests { /// The HA flow: table locations carry a logical authority while /// `hdfs.name-node` carries the (comma-separated) endpoints; it wins. #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_configured_name_node() { set_up(); let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Hdfs)) @@ -106,7 +103,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_overwrite() { let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_overwrite"); @@ -128,7 +124,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_delete_stream() { let file_io = get_file_io(); @@ -155,7 +150,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_delete_stream_empty() { let file_io = get_file_io(); let stream = futures::stream::empty().boxed(); @@ -163,7 +157,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_resolving_storage() { set_up(); let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())).build(); @@ -186,7 +179,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_metadata() { let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_metadata"); @@ -205,7 +197,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_delete() { let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_delete"); @@ -223,7 +214,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_delete_prefix() { let file_io = get_file_io(); let dir = test_path("test_file_io_hdfs_delete_prefix"); @@ -248,7 +238,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_reader_range() { let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_reader_range"); @@ -274,7 +263,6 @@ mod tests { } #[tokio::test] - #[ignore = "Linux-only: HDFS docker fixture uses host networking"] async fn test_file_io_hdfs_streaming_writer() { let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_streaming_writer"); diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 6416e8983f..485adb900a 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -151,7 +151,7 @@ services: # HDFS - single-node NameNode + DataNode for HDFS tests # ============================================================================= # hdfs-native connects to DataNodes by their registered IP โ€” unroutable on - # a docker bridge, hence host networking (Linux-only; tests are `#[ignore]`d). + # a docker bridge, hence host networking (needs Linux or a recent runtime). hdfs-namenode: image: apache/hadoop:3.5.0 network_mode: "host" From 45bd68a8d3ab21f867e07453c389be51416ec962 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Mon, 31 Aug 2026 12:23:03 +0300 Subject: [PATCH 03/51] fix: reject non-hierarchical hdfs URLs; key delete_stream batches by effective NameNode - Url::parse accepts non-hierarchical forms like `hdfs:x`; the byte-7 slice then panicked. Require the literal `hdfs://` prefix instead. - batch_key_for_path grouped by URL host only, so NameNodes differing by port shared one deleter; key by the effective NameNode (configured hdfs.name-node, else authority incl. port), matching the operator cache. Co-Authored-By: Claude Opus 5 --- crates/storage/opendal/src/hdfs.rs | 66 ++++++++++++++++++++++++++++-- crates/storage/opendal/src/lib.rs | 4 ++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/crates/storage/opendal/src/hdfs.rs b/crates/storage/opendal/src/hdfs.rs index 1c0dfd3da4..9abfa70251 100644 --- a/crates/storage/opendal/src/hdfs.rs +++ b/crates/storage/opendal/src/hdfs.rs @@ -59,12 +59,14 @@ pub(crate) fn parse_hdfs_path(path: &str) -> Result<(Option, &str)> { format!("Invalid hdfs path: {path}: {e}"), ) })?; - if url.scheme() != "hdfs" { + // Non-special schemes parse even without `//` (e.g. `hdfs:x` is a valid + // non-hierarchical URL), so require the literal prefix before slicing. + let (Some(after_scheme), "hdfs") = (path.strip_prefix("hdfs://"), url.scheme()) else { return Err(Error::new( ErrorKind::DataInvalid, format!("Invalid hdfs path: {path}, expected scheme `hdfs://`"), )); - } + }; let name_node = url.host_str().filter(|h| !h.is_empty()).map(|host| { url.port() @@ -76,7 +78,6 @@ pub(crate) fn parse_hdfs_path(path: &str) -> Result<(Option, &str)> { // lifetime. Slice the path component out of the original input instead; // it starts after the first `/` following the `hdfs://` prefix. Opendal // paths must not start with `/` (`Deleter::delete` rejects them). - let after_scheme = &path["hdfs://".len()..]; let rel = match after_scheme.find('/') { Some(i) => after_scheme[i..].trim_start_matches('/'), None => "", @@ -134,6 +135,21 @@ pub(crate) fn hdfs_create_operator<'a>( Ok((op, relative_path)) } +/// Returns the `delete_stream` grouping key for a path: the effective +/// NameNode, mirroring the operator-cache key so paths that resolve to +/// different operators never share a deleter. +pub(crate) fn hdfs_batch_key(config: &HdfsNativeConfig, path: &str) -> String { + config + .name_node + .clone() + .or_else(|| { + parse_hdfs_path(path) + .ok() + .and_then(|(name_node, _)| name_node) + }) + .unwrap_or_default() +} + /// Build a new OpenDAL [`Operator`]: OpenDAL splits `name_node` on commas /// into a synthetic HA name service; `$HADOOP_CONF_DIR` XML still merges in. fn hdfs_operator_build(config: &HdfsNativeConfig, name_node: &str) -> Result { @@ -236,6 +252,50 @@ mod tests { assert!(err.to_string().contains("Invalid hdfs path")); } + #[test] + fn test_parse_hdfs_path_non_hierarchical_errors() { + // `hdfs:x` parses as a valid non-hierarchical URL; it must be + // rejected rather than panic on slicing. + for path in ["hdfs:x", "hdfs:/x", "hdfs:"] { + let err = parse_hdfs_path(path).unwrap_err(); + assert!(err.to_string().contains("expected scheme `hdfs://`")); + } + } + + #[test] + fn test_hdfs_batch_key_distinguishes_ports() { + let config = HdfsNativeConfig::default(); + + assert_eq!( + hdfs_batch_key(&config, "hdfs://namenode:8020/a"), + "hdfs://namenode:8020" + ); + assert_eq!( + hdfs_batch_key(&config, "hdfs://namenode:9000/b"), + "hdfs://namenode:9000" + ); + } + + #[test] + fn test_hdfs_batch_key_configured_name_node_wins() { + let config = hdfs_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), + )])) + .unwrap(); + + // All paths group under the configured NameNode, matching the + // single cached operator they resolve to. + assert_eq!( + hdfs_batch_key(&config, "hdfs://ns-a/x"), + "hdfs://nn1:8020,hdfs://nn2:8020" + ); + assert_eq!( + hdfs_batch_key(&config, "hdfs:///y"), + "hdfs://nn1:8020,hdfs://nn2:8020" + ); + } + #[test] fn test_hdfs_create_operator_configured_name_node_wins() { let config = hdfs_config_parse(HashMap::from([( diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 6626c356e1..9e01793077 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -413,6 +413,10 @@ impl OpenDalStorage { match self { #[cfg(feature = "opendal-hf")] OpenDalStorage::Hf { .. } => hf_batch_key(path), + // The URL host alone would merge distinct NameNodes that differ + // only by port; key by the effective NameNode instead. + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorage::Hdfs { config, .. } => hdfs_batch_key(config, path), _ => url::Url::parse(path) .ok() .and_then(|u| u.host_str().map(|s| s.to_string())) From a4178fe023ab5e07f1dcda4b088cd72075e010de Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 5 Sep 2026 12:30:47 +0300 Subject: [PATCH 04/51] refactor(storage): rename hdfs module to hdfs_native, drop unused HdfsConfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: the opendal module, its functions and the storage/factory variants are now hdfs_native-prefixed (leaving room for a libhdfs-backed variant, see #1130), and the never-consumed HdfsConfig struct is removed from the core crate โ€” config/hdfs.rs keeps only the property constants that iceberg-storage-opendal uses. Co-Authored-By: Claude Fable 5.1 --- crates/iceberg/src/io/storage/config/hdfs.rs | 53 ----------- crates/storage/opendal/public-api.txt | 8 +- .../opendal/src/{hdfs.rs => hdfs_native.rs} | 93 ++++++++++--------- crates/storage/opendal/src/lib.rs | 38 ++++---- crates/storage/opendal/src/resolving.rs | 14 +-- .../opendal/tests/file_io_hdfs_test.rs | 4 +- 6 files changed, 79 insertions(+), 131 deletions(-) rename crates/storage/opendal/src/{hdfs.rs => hdfs_native.rs} (73%) diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index e2d468dec1..626f23ffb5 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -17,10 +17,6 @@ //! HDFS storage configuration. -use std::collections::HashMap; - -use iceberg_property_macro::Properties; - /// HDFS NameNode RPC endpoint(s), e.g. `hdfs://namenode:8020`; a /// comma-separated list enables HA failover. When unset, the NameNode is /// derived from the path authority. @@ -29,52 +25,3 @@ pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; /// `hadoop.dfs.client.failover.random.order`. Forwarded values (prefix /// stripped) override those loaded from `$HADOOP_CONF_DIR`. pub const HDFS_HADOOP_CONF_PREFIX: &str = "hadoop."; - -/// HDFS storage configuration. -// No in-crate consumer yet: `iceberg-storage-opendal` parses the raw -// properties itself and only shares the key constants above. -#[allow(dead_code)] -#[derive(Debug, Properties)] -pub(crate) struct HdfsConfig { - /// NameNode endpoint(s); comma-separated for HA failover. - #[property(key = HDFS_NAME_NODE, default = None, getter)] - name_node: Option, - /// Extra HDFS client configuration (the `hadoop.` prefix stripped). - #[property(prefix = HDFS_HADOOP_CONF_PREFIX, getter)] - options: HashMap, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_hdfs_config_from_properties() { - let props = HashMap::from([ - ( - HDFS_NAME_NODE.to_string(), - "hdfs://namenode:8020".to_string(), - ), - ( - "hadoop.dfs.client.failover.random.order".to_string(), - "true".to_string(), - ), - ("unrelated.key".to_string(), "ignored".to_string()), - ]); - - let cfg = HdfsConfig::from_properties(&props).unwrap(); - assert_eq!(cfg.name_node().as_deref(), Some("hdfs://namenode:8020")); - assert_eq!( - cfg.options().get("dfs.client.failover.random.order"), - Some(&"true".to_string()) - ); - assert!(!cfg.options().contains_key("unrelated.key")); - } - - #[test] - fn test_hdfs_config_empty() { - let cfg = HdfsConfig::from_properties(&HashMap::new()).unwrap(); - assert_eq!(cfg.name_node().as_deref(), None); - assert!(cfg.options().is_empty()); - } -} diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index 2a562eaaf3..58bd9ee6b7 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -6,9 +6,9 @@ pub iceberg_storage_opendal::OpenDalStorage::Azdls pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::Gcs pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::Hdfs -pub iceberg_storage_opendal::OpenDalStorage::Hdfs::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::Hdfs::operators: alloc::sync::Arc>> +pub iceberg_storage_opendal::OpenDalStorage::HdfsNative +pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::config: alloc::sync::Arc +pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::operators: alloc::sync::Arc>> pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::LocalFs @@ -42,7 +42,7 @@ pub enum iceberg_storage_opendal::OpenDalStorageFactory pub iceberg_storage_opendal::OpenDalStorageFactory::Azdls pub iceberg_storage_opendal::OpenDalStorageFactory::Fs pub iceberg_storage_opendal::OpenDalStorageFactory::Gcs -pub iceberg_storage_opendal::OpenDalStorageFactory::Hdfs +pub iceberg_storage_opendal::OpenDalStorageFactory::HdfsNative pub iceberg_storage_opendal::OpenDalStorageFactory::Hf pub iceberg_storage_opendal::OpenDalStorageFactory::Memory pub iceberg_storage_opendal::OpenDalStorageFactory::Oss diff --git a/crates/storage/opendal/src/hdfs.rs b/crates/storage/opendal/src/hdfs_native.rs similarity index 73% rename from crates/storage/opendal/src/hdfs.rs rename to crates/storage/opendal/src/hdfs_native.rs index 9abfa70251..9de9ecab5f 100644 --- a/crates/storage/opendal/src/hdfs.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -29,7 +29,7 @@ use url::Url; use crate::utils::from_opendal_error; /// Parse iceberg properties to [`HdfsNativeConfig`]. -pub(crate) fn hdfs_config_parse(mut m: HashMap) -> Result { +pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result { let mut cfg = HdfsNativeConfig::default(); if let Some(name_node) = m.remove(HDFS_NAME_NODE) { @@ -52,7 +52,7 @@ pub(crate) fn hdfs_config_parse(mut m: HashMap) -> Result")` (`None` when /// authority-less) and the relative path (no leading `/`, opendal style). -pub(crate) fn parse_hdfs_path(path: &str) -> Result<(Option, &str)> { +pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str)> { let url = Url::parse(path).map_err(|e| { Error::new( ErrorKind::DataInvalid, @@ -89,12 +89,12 @@ pub(crate) fn parse_hdfs_path(path: &str) -> Result<(Option, &str)> { /// Creates an operator for the path, cached per effective NameNode (the /// configured `hdfs.name-node`, else the path authority) โ€” each operator /// holds an HDFS client with live RPC connections. -pub(crate) fn hdfs_create_operator<'a>( +pub(crate) fn hdfs_native_create_operator<'a>( path: &'a str, config: &HdfsNativeConfig, operators: &RwLock>, ) -> Result<(Operator, &'a str)> { - let (authority_name_node, relative_path) = parse_hdfs_path(path)?; + let (authority_name_node, relative_path) = hdfs_native_parse_path(path)?; let name_node = match config.name_node.clone().or(authority_name_node) { Some(name_node) => name_node, @@ -126,7 +126,7 @@ pub(crate) fn hdfs_create_operator<'a>( let op = match cache.get(&name_node) { Some(op) => op.clone(), None => { - let op = hdfs_operator_build(config, &name_node)?; + let op = hdfs_native_operator_build(config, &name_node)?; cache.insert(name_node, op.clone()); op } @@ -138,12 +138,12 @@ pub(crate) fn hdfs_create_operator<'a>( /// Returns the `delete_stream` grouping key for a path: the effective /// NameNode, mirroring the operator-cache key so paths that resolve to /// different operators never share a deleter. -pub(crate) fn hdfs_batch_key(config: &HdfsNativeConfig, path: &str) -> String { +pub(crate) fn hdfs_native_batch_key(config: &HdfsNativeConfig, path: &str) -> String { config .name_node .clone() .or_else(|| { - parse_hdfs_path(path) + hdfs_native_parse_path(path) .ok() .and_then(|(name_node, _)| name_node) }) @@ -152,7 +152,7 @@ pub(crate) fn hdfs_batch_key(config: &HdfsNativeConfig, path: &str) -> String { /// Build a new OpenDAL [`Operator`]: OpenDAL splits `name_node` on commas /// into a synthetic HA name service; `$HADOOP_CONF_DIR` XML still merges in. -fn hdfs_operator_build(config: &HdfsNativeConfig, name_node: &str) -> Result { +fn hdfs_native_operator_build(config: &HdfsNativeConfig, name_node: &str) -> Result { let mut cfg = config.clone(); cfg.name_node = Some(name_node.to_string()); Operator::from_config(cfg).map_err(from_opendal_error) @@ -163,7 +163,7 @@ mod tests { use super::*; #[test] - fn test_hdfs_config_parse_name_node_and_options() { + fn test_hdfs_native_config_parse_name_node_and_options() { let props = HashMap::from([ ( HDFS_NAME_NODE.to_string(), @@ -176,7 +176,7 @@ mod tests { ("unrelated.key".to_string(), "ignored".to_string()), ]); - let cfg = hdfs_config_parse(props).unwrap(); + let cfg = hdfs_native_config_parse(props).unwrap(); assert_eq!( cfg.name_node.as_deref(), @@ -191,94 +191,94 @@ mod tests { } #[test] - fn test_hdfs_config_parse_empty() { - let cfg = hdfs_config_parse(HashMap::new()).unwrap(); + fn test_hdfs_native_config_parse_empty() { + let cfg = hdfs_native_config_parse(HashMap::new()).unwrap(); assert_eq!(cfg.name_node, None); assert_eq!(cfg.options, None); } #[test] - fn test_parse_hdfs_path_with_authority_and_rel() { - let (nn, rel) = parse_hdfs_path("hdfs://nameservice1/a/b").unwrap(); + fn test_hdfs_native_parse_path_with_authority_and_rel() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1/a/b").unwrap(); assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); assert_eq!(rel, "a/b"); } #[test] - fn test_parse_hdfs_path_with_authority_and_port() { - let (nn, rel) = parse_hdfs_path("hdfs://nn:8020/foo").unwrap(); + fn test_hdfs_native_parse_path_with_authority_and_port() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nn:8020/foo").unwrap(); assert_eq!(nn.as_deref(), Some("hdfs://nn:8020")); assert_eq!(rel, "foo"); } #[test] - fn test_parse_hdfs_path_with_authority_no_path() { - let (nn, rel) = parse_hdfs_path("hdfs://nameservice1").unwrap(); + fn test_hdfs_native_parse_path_with_authority_no_path() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1").unwrap(); assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); assert_eq!(rel, ""); } #[test] - fn test_parse_hdfs_path_with_authority_trailing_slash() { - let (nn, rel) = parse_hdfs_path("hdfs://nameservice1/").unwrap(); + fn test_hdfs_native_parse_path_with_authority_trailing_slash() { + let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1/").unwrap(); assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); assert_eq!(rel, ""); } #[test] - fn test_parse_hdfs_path_authority_less_returns_none() { - let (nn, rel) = parse_hdfs_path("hdfs:///a/b").unwrap(); + fn test_hdfs_native_parse_path_authority_less_returns_none() { + let (nn, rel) = hdfs_native_parse_path("hdfs:///a/b").unwrap(); assert_eq!(nn, None); assert_eq!(rel, "a/b"); } #[test] - fn test_parse_hdfs_path_wrong_scheme_errors() { - let err = parse_hdfs_path("file:///tmp/x").unwrap_err(); + fn test_hdfs_native_parse_path_wrong_scheme_errors() { + let err = hdfs_native_parse_path("file:///tmp/x").unwrap_err(); assert!(err.to_string().contains("expected scheme `hdfs://`")); } #[test] - fn test_parse_hdfs_path_invalid_url_errors() { - let err = parse_hdfs_path("not-a-url").unwrap_err(); + fn test_hdfs_native_parse_path_invalid_url_errors() { + let err = hdfs_native_parse_path("not-a-url").unwrap_err(); assert!(err.to_string().contains("Invalid hdfs path")); } #[test] - fn test_parse_hdfs_path_non_hierarchical_errors() { + fn test_hdfs_native_parse_path_non_hierarchical_errors() { // `hdfs:x` parses as a valid non-hierarchical URL; it must be // rejected rather than panic on slicing. for path in ["hdfs:x", "hdfs:/x", "hdfs:"] { - let err = parse_hdfs_path(path).unwrap_err(); + let err = hdfs_native_parse_path(path).unwrap_err(); assert!(err.to_string().contains("expected scheme `hdfs://`")); } } #[test] - fn test_hdfs_batch_key_distinguishes_ports() { + fn test_hdfs_native_batch_key_distinguishes_ports() { let config = HdfsNativeConfig::default(); assert_eq!( - hdfs_batch_key(&config, "hdfs://namenode:8020/a"), + hdfs_native_batch_key(&config, "hdfs://namenode:8020/a"), "hdfs://namenode:8020" ); assert_eq!( - hdfs_batch_key(&config, "hdfs://namenode:9000/b"), + hdfs_native_batch_key(&config, "hdfs://namenode:9000/b"), "hdfs://namenode:9000" ); } #[test] - fn test_hdfs_batch_key_configured_name_node_wins() { - let config = hdfs_config_parse(HashMap::from([( + fn test_hdfs_native_batch_key_configured_name_node_wins() { + let config = hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), )])) @@ -287,18 +287,18 @@ mod tests { // All paths group under the configured NameNode, matching the // single cached operator they resolve to. assert_eq!( - hdfs_batch_key(&config, "hdfs://ns-a/x"), + hdfs_native_batch_key(&config, "hdfs://ns-a/x"), "hdfs://nn1:8020,hdfs://nn2:8020" ); assert_eq!( - hdfs_batch_key(&config, "hdfs:///y"), + hdfs_native_batch_key(&config, "hdfs:///y"), "hdfs://nn1:8020,hdfs://nn2:8020" ); } #[test] - fn test_hdfs_create_operator_configured_name_node_wins() { - let config = hdfs_config_parse(HashMap::from([( + fn test_hdfs_native_create_operator_configured_name_node_wins() { + let config = hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), "hdfs://configured:8020".to_string(), )])) @@ -306,7 +306,7 @@ mod tests { let operators = RwLock::new(HashMap::new()); let (_, rel) = - hdfs_create_operator("hdfs://from-path:9000/a/b", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs://from-path:9000/a/b", &config, &operators).unwrap(); assert_eq!(rel, "a/b"); let cache = operators.read().unwrap(); @@ -315,34 +315,35 @@ mod tests { } #[test] - fn test_hdfs_create_operator_uses_path_authority() { + fn test_hdfs_native_create_operator_uses_path_authority() { let config = HdfsNativeConfig::default(); let operators = RwLock::new(HashMap::new()); - let (_, rel) = hdfs_create_operator("hdfs://nn:8020/a/b", &config, &operators).unwrap(); + let (_, rel) = + hdfs_native_create_operator("hdfs://nn:8020/a/b", &config, &operators).unwrap(); assert_eq!(rel, "a/b"); assert!(operators.read().unwrap().contains_key("hdfs://nn:8020")); } #[test] - fn test_hdfs_create_operator_caches_per_name_node() { + fn test_hdfs_native_create_operator_caches_per_name_node() { let config = HdfsNativeConfig::default(); let operators = RwLock::new(HashMap::new()); - hdfs_create_operator("hdfs://nn1:8020/a", &config, &operators).unwrap(); - hdfs_create_operator("hdfs://nn1:8020/b", &config, &operators).unwrap(); - hdfs_create_operator("hdfs://nn2:8020/c", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs://nn1:8020/a", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs://nn1:8020/b", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs://nn2:8020/c", &config, &operators).unwrap(); assert_eq!(operators.read().unwrap().len(), 2); } #[test] - fn test_hdfs_create_operator_authority_less_without_config_errors() { + fn test_hdfs_native_create_operator_authority_less_without_config_errors() { let config = HdfsNativeConfig::default(); let operators = RwLock::new(HashMap::new()); - let err = hdfs_create_operator("hdfs:///a/b", &config, &operators).unwrap_err(); + let err = hdfs_native_create_operator("hdfs:///a/b", &config, &operators).unwrap_err(); assert!(err.to_string().contains(HDFS_NAME_NODE)); } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 0defd6dea9..d3bec9d23f 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -76,10 +76,10 @@ cfg_if! { cfg_if! { if #[cfg(feature = "opendal-hdfs-native")] { - mod hdfs; + mod hdfs_native; use std::sync::RwLock; - use hdfs::*; + use hdfs_native::*; use opendal::services::HdfsNativeConfig; } } @@ -147,7 +147,7 @@ pub enum OpenDalStorageFactory { Gcs, /// HDFS storage factory. #[cfg(feature = "opendal-hdfs-native")] - Hdfs, + HdfsNative, /// OSS storage factory. #[cfg(feature = "opendal-oss")] Oss, @@ -195,8 +195,8 @@ impl StorageFactory for OpenDalStorageFactory { config: gcs_config_parse(config.props().clone())?.into(), })), #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorageFactory::Hdfs => Ok(Arc::new(OpenDalStorage::Hdfs { - config: hdfs_config_parse(config.props().clone())?.into(), + OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative { + config: hdfs_native_config_parse(config.props().clone())?.into(), operators: Arc::new(RwLock::new(HashMap::new())), })), #[cfg(feature = "opendal-oss")] @@ -267,7 +267,7 @@ pub enum OpenDalStorage { /// The NameNode is taken from the `hdfs.name-node` property when set /// (comma-separated endpoints enable HA failover), else the path authority. #[cfg(feature = "opendal-hdfs-native")] - Hdfs { + HdfsNative { /// HDFS configuration. config: Arc, /// Operator cache keyed by effective NameNode. @@ -380,8 +380,8 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::Hdfs { config, operators } => { - hdfs_create_operator(path, config, operators)? + OpenDalStorage::HdfsNative { config, operators } => { + hdfs_native_create_operator(path, config, operators)? } #[cfg(feature = "opendal-oss")] OpenDalStorage::Oss { config } => { @@ -442,7 +442,7 @@ impl OpenDalStorage { // The URL host alone would merge distinct NameNodes that differ // only by port; key by the effective NameNode instead. #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::Hdfs { config, .. } => hdfs_batch_key(config, path), + OpenDalStorage::HdfsNative { config, .. } => hdfs_native_batch_key(config, path), _ => url::Url::parse(path) .ok() .and_then(|u| u.host_str().map(|s| s.to_string())) @@ -501,8 +501,8 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::Hdfs { .. } => { - let (_, relative_path) = parse_hdfs_path(path)?; + OpenDalStorage::HdfsNative { .. } => { + let (_, relative_path) = hdfs_native_parse_path(path)?; Ok(relative_path) } #[cfg(feature = "opendal-oss")] @@ -854,8 +854,8 @@ mod tests { } #[cfg(feature = "opendal-hdfs-native")] - fn hdfs_test_storage() -> OpenDalStorage { - OpenDalStorage::Hdfs { + fn hdfs_native_test_storage() -> OpenDalStorage { + OpenDalStorage::HdfsNative { config: Arc::new(HdfsNativeConfig::default()), operators: Arc::new(RwLock::new(HashMap::new())), } @@ -863,8 +863,8 @@ mod tests { #[cfg(feature = "opendal-hdfs-native")] #[test] - fn test_relativize_path_hdfs() { - let storage = hdfs_test_storage(); + fn test_relativize_path_hdfs_native() { + let storage = hdfs_native_test_storage(); assert_eq!( storage @@ -882,16 +882,16 @@ mod tests { #[cfg(feature = "opendal-hdfs-native")] #[test] - fn test_relativize_path_hdfs_authority_less() { - let storage = hdfs_test_storage(); + fn test_relativize_path_hdfs_native_authority_less() { + let storage = hdfs_native_test_storage(); assert_eq!(storage.relativize_path("hdfs:///a/b").unwrap(), "a/b"); } #[cfg(feature = "opendal-hdfs-native")] #[test] - fn test_relativize_path_hdfs_wrong_scheme_errors() { - let storage = hdfs_test_storage(); + fn test_relativize_path_hdfs_native_wrong_scheme_errors() { + let storage = hdfs_native_test_storage(); assert!(storage.relativize_path("s3://bucket/x").is_err()); } diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 8a15ebd41f..b3af99928e 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -131,8 +131,8 @@ fn build_storage_for_scheme( } #[cfg(feature = "opendal-hdfs-native")] "hdfs" => { - let config = crate::hdfs::hdfs_config_parse(props.clone())?; - Ok(OpenDalStorage::Hdfs { + let config = crate::hdfs_native::hdfs_native_config_parse(props.clone())?; + Ok(OpenDalStorage::HdfsNative { config: Arc::new(config), operators: Arc::new(RwLock::new(HashMap::new())), }) @@ -405,20 +405,20 @@ mod tests { #[cfg(feature = "opendal-hdfs-native")] #[test] - fn test_resolve_hdfs_returns_hdfs_variant() { + fn test_resolve_hdfs_native_returns_hdfs_variant() { let storage = empty_resolving_storage(); let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap(); assert!( - matches!(&*resolved, OpenDalStorage::Hdfs { .. }), - "expected Hdfs variant, got {resolved:?}" + matches!(&*resolved, OpenDalStorage::HdfsNative { .. }), + "expected HdfsNative variant, got {resolved:?}" ); } #[cfg(feature = "opendal-hdfs-native")] #[test] - fn test_resolve_hdfs_distinct_authorities_share_instance() { + fn test_resolve_hdfs_native_distinct_authorities_share_instance() { let storage = empty_resolving_storage(); let a = storage.resolve("hdfs://ns1/a").unwrap(); @@ -426,7 +426,7 @@ mod tests { assert!( Arc::ptr_eq(&a, &b), - "different authorities should share the OpenDalStorage::Hdfs instance (operator cache is internal)" + "different authorities should share the OpenDalStorage::HdfsNative instance (operator cache is internal)" ); } diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index f072a9c476..d96c2eba81 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -33,7 +33,7 @@ mod tests { fn get_file_io() -> FileIO { set_up(); - FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Hdfs)).build() + FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)).build() } fn test_path(suffix: &str) -> String { @@ -77,7 +77,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_configured_name_node() { set_up(); - let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Hdfs)) + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop(HDFS_NAME_NODE, get_hdfs_endpoint()) .build(); From 9d7d2d89e8391245863ebbcbad753b99a0b3b4cc Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sun, 13 Sep 2026 11:07:30 +0300 Subject: [PATCH 05/51] ci: retrigger after setup-python manifest timeout on macos runner Co-Authored-By: Claude Fable 5.1 From d17ae3882e82977e8f9786e287edca07310c3cf6 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Thu, 17 Sep 2026 11:36:54 +0300 Subject: [PATCH 06/51] =?UTF-8?q?fix(storage):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20cache=20newtype,=20build=20outside=20lock,=20single?= =?UTF-8?q?=20NameNode=20rule,=20env-gated=20HDFS=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HdfsNativeOperatorCache newtype keeps the cache representation out of the public API. - Operators are built outside the cache lock (the build reads Hadoop XML synchronously); a racing first caller's duplicate is dropped unopened. - hdfs_native_effective_name_node is the single source of the configured-else-authority rule for both create_operator and the delete_stream batch key; unresolvable paths key on themselves. - hdfs.name-node is trimmed and an empty value is treated as unset, so it no longer shadows the path-authority fallback. - HDFS integration tests self-skip unless ICEBERG_TEST_HDFS_ENDPOINT is set (as the HF tests do) and the compose services sit behind the hdfs profile; the Linux CI job opts in via env. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 6 + crates/storage/opendal/public-api.txt | 9 +- crates/storage/opendal/src/hdfs_native.rs | 211 +++++++++++------- crates/storage/opendal/src/lib.rs | 9 +- crates/storage/opendal/src/resolving.rs | 2 +- .../opendal/tests/file_io_hdfs_test.rs | 43 +++- dev/docker-compose.yaml | 3 + 7 files changed, 197 insertions(+), 86 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9fce8c20c..24031c1109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -241,6 +241,9 @@ jobs: - name: Start Docker containers if: matrix.test-suite.name == 'default' + env: + # The HDFS fixture needs host networking; opt in here (Linux only). + COMPOSE_PROFILES: hdfs run: make docker-up - name: Run tests @@ -251,6 +254,9 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_BUCKET: ${{ secrets.HF_BUCKET }} HF_DATASET: ${{ secrets.HF_DATASET }} + # HDFS integration tests self-skip when unset; see + # crates/storage/opendal/tests/file_io_hdfs_test.rs + ICEBERG_TEST_HDFS_ENDPOINT: hdfs://localhost:8020 run: | if [ "${{ matrix.test-suite.name }}" = "default" ]; then cargo nextest run ${{ matrix.test-suite.args }} diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index 58bd9ee6b7..b5ccaf078c 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -8,7 +8,7 @@ pub iceberg_storage_opendal::OpenDalStorage::Gcs pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::HdfsNative pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::operators: alloc::sync::Arc>> +pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::operators: iceberg_storage_opendal::HdfsNativeOperatorCache pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc pub iceberg_storage_opendal::OpenDalStorage::LocalFs @@ -65,6 +65,13 @@ impl core::clone::Clone for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::clone(&self) -> Self impl core::fmt::Debug for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_storage_opendal::HdfsNativeOperatorCache(_) +impl core::clone::Clone for iceberg_storage_opendal::HdfsNativeOperatorCache +pub fn iceberg_storage_opendal::HdfsNativeOperatorCache::clone(&self) -> iceberg_storage_opendal::HdfsNativeOperatorCache +impl core::default::Default for iceberg_storage_opendal::HdfsNativeOperatorCache +pub fn iceberg_storage_opendal::HdfsNativeOperatorCache::default() -> iceberg_storage_opendal::HdfsNativeOperatorCache +impl core::fmt::Debug for iceberg_storage_opendal::HdfsNativeOperatorCache +pub fn iceberg_storage_opendal::HdfsNativeOperatorCache::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result pub struct iceberg_storage_opendal::OpenDalResolvingStorage impl core::fmt::Debug for iceberg_storage_opendal::OpenDalResolvingStorage pub fn iceberg_storage_opendal::OpenDalResolvingStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 9de9ecab5f..48d68f80ab 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -18,7 +18,7 @@ //! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). use std::collections::HashMap; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_NAME_NODE}; use iceberg::{Error, ErrorKind, Result}; @@ -32,7 +32,13 @@ use crate::utils::from_opendal_error; pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result { let mut cfg = HdfsNativeConfig::default(); - if let Some(name_node) = m.remove(HDFS_NAME_NODE) { + // `Operator::from_config` bypasses the builder's empty-string guard, and + // `Some("")` would shadow the path-authority fallback below. + if let Some(name_node) = m + .remove(HDFS_NAME_NODE) + .map(|s| s.trim().trim_end_matches('/').to_string()) + .filter(|s| !s.is_empty()) + { cfg.name_node = Some(name_node); } @@ -86,68 +92,91 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str Ok((name_node, rel)) } -/// Creates an operator for the path, cached per effective NameNode (the -/// configured `hdfs.name-node`, else the path authority) โ€” each operator -/// holds an HDFS client with live RPC connections. -pub(crate) fn hdfs_native_create_operator<'a>( - path: &'a str, +/// Resolves the effective NameNode for a path โ€” the configured +/// `hdfs.name-node` when set, else the path authority โ€” plus the relative +/// path. Both the operator cache and `delete_stream` batching key on this, +/// so they cannot drift apart. +pub(crate) fn hdfs_native_effective_name_node<'a>( config: &HdfsNativeConfig, - operators: &RwLock>, -) -> Result<(Operator, &'a str)> { + path: &'a str, +) -> Result<(String, &'a str)> { let (authority_name_node, relative_path) = hdfs_native_parse_path(path)?; - - let name_node = match config.name_node.clone().or(authority_name_node) { - Some(name_node) => name_node, - None => { - return Err(Error::new( + let name_node = config + .name_node + .clone() + .or(authority_name_node) + .ok_or_else(|| { + Error::new( ErrorKind::DataInvalid, format!( "Invalid hdfs path: {path}, authority-less paths require the `{HDFS_NAME_NODE}` property" ), - )); - } - }; + ) + })?; + Ok((name_node, relative_path)) +} - // Fast path: check read lock first. - { - let cache = operators - .read() - .map_err(|_| Error::new(ErrorKind::Unexpected, "HDFS operator cache lock poisoned"))?; - if let Some(op) = cache.get(&name_node) { - return Ok((op.clone(), relative_path)); - } +/// Operators cached per effective NameNode: each holds an `hdfs-native` +/// client with live RPC connections, whose tasks run on the tokio runtime +/// current when it was built. +#[derive(Clone, Debug, Default)] +pub struct HdfsNativeOperatorCache(Arc>>); + +impl HdfsNativeOperatorCache { + fn get(&self, name_node: &str) -> Result> { + Ok(self.0.read().map_err(poisoned)?.get(name_node).cloned()) } - // Slow path: build and insert under write lock, re-checking for a - // concurrent insert. - let mut cache = operators - .write() - .map_err(|_| Error::new(ErrorKind::Unexpected, "HDFS operator cache lock poisoned"))?; - let op = match cache.get(&name_node) { - Some(op) => op.clone(), - None => { - let op = hdfs_native_operator_build(config, &name_node)?; - cache.insert(name_node, op.clone()); - op - } - }; + /// Inserts `op` unless a concurrent caller got there first, returning + /// whichever operator the cache now holds. + fn insert(&self, name_node: String, op: Operator) -> Result { + Ok(self + .0 + .write() + .map_err(poisoned)? + .entry(name_node) + .or_insert(op) + .clone()) + } + + #[cfg(test)] + fn len(&self) -> usize { + self.0.read().unwrap().len() + } +} + +fn poisoned(_: T) -> Error { + Error::new(ErrorKind::Unexpected, "HDFS operator cache lock poisoned") +} + +/// Creates an operator for the path, reusing the cached one for its +/// effective NameNode. +pub(crate) fn hdfs_native_create_operator<'a>( + path: &'a str, + config: &HdfsNativeConfig, + operators: &HdfsNativeOperatorCache, +) -> Result<(Operator, &'a str)> { + let (name_node, relative_path) = hdfs_native_effective_name_node(config, path)?; + + if let Some(op) = operators.get(&name_node)? { + return Ok((op, relative_path)); + } - Ok((op, relative_path)) + // Built outside the lock: the build reads the Hadoop XML config + // synchronously. A racing first caller may build too; the loser is + // dropped before opening any connection. + let op = hdfs_native_operator_build(config, &name_node)?; + Ok((operators.insert(name_node, op)?, relative_path)) } /// Returns the `delete_stream` grouping key for a path: the effective -/// NameNode, mirroring the operator-cache key so paths that resolve to -/// different operators never share a deleter. +/// NameNode, so paths that resolve to different operators never share a +/// deleter. Unresolvable paths key on themselves (as `hf_batch_key` does); +/// `create_operator` then reports the real error. pub(crate) fn hdfs_native_batch_key(config: &HdfsNativeConfig, path: &str) -> String { - config - .name_node - .clone() - .or_else(|| { - hdfs_native_parse_path(path) - .ok() - .and_then(|(name_node, _)| name_node) - }) - .unwrap_or_default() + hdfs_native_effective_name_node(config, path) + .map(|(name_node, _)| name_node) + .unwrap_or_else(|_| path.to_string()) } /// Build a new OpenDAL [`Operator`]: OpenDAL splits `name_node` on commas @@ -198,6 +227,50 @@ mod tests { assert_eq!(cfg.options, None); } + #[test] + fn test_hdfs_native_config_parse_normalizes_name_node() { + let parse = |value: &str| { + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + value.to_string(), + )])) + .unwrap() + .name_node + }; + + // Empty must not shadow the path-authority fallback. + assert_eq!(parse(""), None); + assert_eq!(parse(" "), None); + // Trailing `/` would otherwise yield a second cache entry for one cluster. + assert_eq!( + parse(" hdfs://nn:8020/ ").as_deref(), + Some("hdfs://nn:8020") + ); + } + + #[test] + fn test_hdfs_native_effective_name_node_precedence() { + let configured = hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), + )])) + .unwrap(); + let unconfigured = HdfsNativeConfig::default(); + + // Configured wins over the authority, including for authority-less paths. + for path in ["hdfs://ns-a/x", "hdfs:///y"] { + let (nn, _) = hdfs_native_effective_name_node(&configured, path).unwrap(); + assert_eq!(nn, "hdfs://nn1:8020,hdfs://nn2:8020"); + } + // Otherwise the authority, including its port. + let (nn, rel) = + hdfs_native_effective_name_node(&unconfigured, "hdfs://nn:9000/a/b").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://nn:9000", "a/b")); + // Neither: a pointed error. + let err = hdfs_native_effective_name_node(&unconfigured, "hdfs:///a").unwrap_err(); + assert!(err.to_string().contains(HDFS_NAME_NODE)); + } + #[test] fn test_hdfs_native_parse_path_with_authority_and_rel() { let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1/a/b").unwrap(); @@ -277,23 +350,12 @@ mod tests { } #[test] - fn test_hdfs_native_batch_key_configured_name_node_wins() { - let config = hdfs_native_config_parse(HashMap::from([( - HDFS_NAME_NODE.to_string(), - "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), - )])) - .unwrap(); + fn test_hdfs_native_batch_key_invalid_path_keys_on_itself() { + let config = HdfsNativeConfig::default(); - // All paths group under the configured NameNode, matching the - // single cached operator they resolve to. - assert_eq!( - hdfs_native_batch_key(&config, "hdfs://ns-a/x"), - "hdfs://nn1:8020,hdfs://nn2:8020" - ); - assert_eq!( - hdfs_native_batch_key(&config, "hdfs:///y"), - "hdfs://nn1:8020,hdfs://nn2:8020" - ); + // Unresolvable paths must not collapse onto a shared "" key. + assert_eq!(hdfs_native_batch_key(&config, "not-a-url"), "not-a-url"); + assert_eq!(hdfs_native_batch_key(&config, "hdfs:///a"), "hdfs:///a"); } #[test] @@ -303,45 +365,44 @@ mod tests { "hdfs://configured:8020".to_string(), )])) .unwrap(); - let operators = RwLock::new(HashMap::new()); + let operators = HdfsNativeOperatorCache::default(); let (_, rel) = hdfs_native_create_operator("hdfs://from-path:9000/a/b", &config, &operators).unwrap(); assert_eq!(rel, "a/b"); - let cache = operators.read().unwrap(); - assert!(cache.contains_key("hdfs://configured:8020")); - assert!(!cache.contains_key("hdfs://from-path:9000")); + assert!(operators.get("hdfs://configured:8020").unwrap().is_some()); + assert!(operators.get("hdfs://from-path:9000").unwrap().is_none()); } #[test] fn test_hdfs_native_create_operator_uses_path_authority() { let config = HdfsNativeConfig::default(); - let operators = RwLock::new(HashMap::new()); + let operators = HdfsNativeOperatorCache::default(); let (_, rel) = hdfs_native_create_operator("hdfs://nn:8020/a/b", &config, &operators).unwrap(); assert_eq!(rel, "a/b"); - assert!(operators.read().unwrap().contains_key("hdfs://nn:8020")); + assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); } #[test] fn test_hdfs_native_create_operator_caches_per_name_node() { let config = HdfsNativeConfig::default(); - let operators = RwLock::new(HashMap::new()); + let operators = HdfsNativeOperatorCache::default(); hdfs_native_create_operator("hdfs://nn1:8020/a", &config, &operators).unwrap(); hdfs_native_create_operator("hdfs://nn1:8020/b", &config, &operators).unwrap(); hdfs_native_create_operator("hdfs://nn2:8020/c", &config, &operators).unwrap(); - assert_eq!(operators.read().unwrap().len(), 2); + assert_eq!(operators.len(), 2); } #[test] fn test_hdfs_native_create_operator_authority_less_without_config_errors() { let config = HdfsNativeConfig::default(); - let operators = RwLock::new(HashMap::new()); + let operators = HdfsNativeOperatorCache::default(); let err = hdfs_native_create_operator("hdfs:///a/b", &config, &operators).unwrap_err(); diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index d3bec9d23f..19a433ca56 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -77,9 +77,8 @@ cfg_if! { cfg_if! { if #[cfg(feature = "opendal-hdfs-native")] { mod hdfs_native; - use std::sync::RwLock; - use hdfs_native::*; + pub use hdfs_native::HdfsNativeOperatorCache; use opendal::services::HdfsNativeConfig; } } @@ -197,7 +196,7 @@ impl StorageFactory for OpenDalStorageFactory { #[cfg(feature = "opendal-hdfs-native")] OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative { config: hdfs_native_config_parse(config.props().clone())?.into(), - operators: Arc::new(RwLock::new(HashMap::new())), + operators: HdfsNativeOperatorCache::default(), })), #[cfg(feature = "opendal-oss")] OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss { @@ -272,7 +271,7 @@ pub enum OpenDalStorage { config: Arc, /// Operator cache keyed by effective NameNode. #[serde(skip, default)] - operators: Arc>>, + operators: HdfsNativeOperatorCache, }, /// OSS storage variant. #[cfg(feature = "opendal-oss")] @@ -857,7 +856,7 @@ mod tests { fn hdfs_native_test_storage() -> OpenDalStorage { OpenDalStorage::HdfsNative { config: Arc::new(HdfsNativeConfig::default()), - operators: Arc::new(RwLock::new(HashMap::new())), + operators: HdfsNativeOperatorCache::default(), } } diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index b3af99928e..5e51dead48 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -134,7 +134,7 @@ fn build_storage_for_scheme( let config = crate::hdfs_native::hdfs_native_config_parse(props.clone())?; Ok(OpenDalStorage::HdfsNative { config: Arc::new(config), - operators: Arc::new(RwLock::new(HashMap::new())), + operators: crate::HdfsNativeOperatorCache::default(), }) } unsupported => Err(Error::new( diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index d96c2eba81..816c784189 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -17,9 +17,16 @@ //! Integration tests for HDFS FileIO via OpenDAL `services-hdfs-native`. //! -//! These tests need the `hdfs-namenode`/`hdfs-datanode` services from -//! `dev/docker-compose.yaml` (started by `make docker-up`); the fixture -//! uses host networking, which needs Linux or a recent Docker runtime. +//! These tests need the HDFS fixture in `dev/docker-compose.yaml` and are +//! skipped when `ICEBERG_TEST_HDFS_ENDPOINT` is not set. The fixture uses +//! host networking (Linux, or a Docker runtime that supports it), so it sits +//! behind a compose profile: +//! +//! ```text +//! COMPOSE_PROFILES=hdfs make docker-up +//! ICEBERG_TEST_HDFS_ENDPOINT=hdfs://localhost:8020 cargo test -p iceberg-storage-opendal \ +//! --features opendal-hdfs-native --test file_io_hdfs_test +//! ``` #[cfg(feature = "opendal-hdfs-native")] mod tests { @@ -29,7 +36,23 @@ mod tests { use futures::StreamExt; use iceberg::io::{FileIO, FileIOBuilder, HDFS_NAME_NODE}; use iceberg_storage_opendal::{OpenDalResolvingStorageFactory, OpenDalStorageFactory}; - use iceberg_test_utils::{get_hdfs_endpoint, normalize_test_name_with_parts, set_up}; + use iceberg_test_utils::{ + ENV_HDFS_ENDPOINT, get_hdfs_endpoint, normalize_test_name_with_parts, set_up, + }; + + /// Skips the calling test unless the HDFS fixture endpoint is configured; + /// an unset *or* empty variable means "not provided" (see the HF tests). + macro_rules! require_hdfs { + () => { + match std::env::var(ENV_HDFS_ENDPOINT) { + Ok(v) if !v.is_empty() => {} + _ => { + eprintln!("Skipping HDFS test: {} not set", ENV_HDFS_ENDPOINT); + return; + } + } + }; + } fn get_file_io() -> FileIO { set_up(); @@ -46,6 +69,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_exists() { + require_hdfs!(); let file_io = get_file_io(); let absent = test_path("test_file_io_hdfs_exists_absent"); @@ -54,6 +78,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_write_and_read() { + require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_write_and_read"); let _ = file_io.delete(&path).await; @@ -76,6 +101,7 @@ mod tests { /// `hdfs.name-node` carries the (comma-separated) endpoints; it wins. #[tokio::test] async fn test_file_io_hdfs_configured_name_node() { + require_hdfs!(); set_up(); let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop(HDFS_NAME_NODE, get_hdfs_endpoint()) @@ -104,6 +130,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_overwrite() { + require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_overwrite"); let _ = file_io.delete(&path).await; @@ -125,6 +152,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete_stream() { + require_hdfs!(); let file_io = get_file_io(); let paths: Vec = (0..5) @@ -151,6 +179,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete_stream_empty() { + require_hdfs!(); let file_io = get_file_io(); let stream = futures::stream::empty().boxed(); file_io.delete_stream(stream).await.unwrap(); @@ -158,6 +187,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_resolving_storage() { + require_hdfs!(); set_up(); let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())).build(); let path = test_path("test_file_io_hdfs_resolving_storage"); @@ -180,6 +210,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_metadata() { + require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_metadata"); let _ = file_io.delete(&path).await; @@ -198,6 +229,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete() { + require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_delete"); @@ -215,6 +247,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete_prefix() { + require_hdfs!(); let file_io = get_file_io(); let dir = test_path("test_file_io_hdfs_delete_prefix"); let _ = file_io.delete_prefix(&dir).await; @@ -239,6 +272,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_reader_range() { + require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_reader_range"); let _ = file_io.delete(&path).await; @@ -264,6 +298,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_streaming_writer() { + require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_streaming_writer"); let _ = file_io.delete(&path).await; diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 15c9df875f..1356470f36 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -152,8 +152,10 @@ services: # ============================================================================= # hdfs-native connects to DataNodes by their registered IP โ€” unroutable on # a docker bridge, hence host networking (needs Linux or a recent runtime). + # Opt-in: `COMPOSE_PROFILES=hdfs make docker-up`. hdfs-namenode: image: apache/hadoop:3.5.0 + profiles: [hdfs] network_mode: "host" command: ["hdfs", "namenode"] environment: @@ -172,6 +174,7 @@ services: hdfs-datanode: image: apache/hadoop:3.5.0 + profiles: [hdfs] network_mode: "host" command: ["hdfs", "datanode"] depends_on: From b8cc860d638a05bdb5f30439a0d1450a1902b0d9 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Thu, 17 Sep 2026 11:36:54 +0300 Subject: [PATCH 07/51] chore(deps): bump rustls to 0.23.45 (RUSTSEC-2026-0285) Inherited from main (tracked in #3222); same bump as #3165. Co-Authored-By: Claude Fable 5.1 --- Cargo.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 237cbb6889..1c117dc921 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -638,9 +638,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -648,9 +648,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -6769,9 +6769,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "once_cell", @@ -6833,9 +6833,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -7818,7 +7818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", From 297afd687f2d0fd16034929574e43b0334252702 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 21:20:21 +0300 Subject: [PATCH 08/51] test(storage): pin IPv6 authority handling in HDFS path parsing Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 48d68f80ab..6e91e14e53 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -287,6 +287,13 @@ mod tests { assert_eq!(rel, "foo"); } + #[test] + fn test_hdfs_native_parse_path_ipv6_authority_keeps_brackets() { + let (nn, rel) = hdfs_native_parse_path("hdfs://[::1]:8020/a/b").unwrap(); + assert_eq!(nn.as_deref(), Some("hdfs://[::1]:8020")); + assert_eq!(rel, "a/b"); + } + #[test] fn test_hdfs_native_parse_path_with_authority_no_path() { let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1").unwrap(); From c018cd161c451deec4ba28d8cb48a3fb0689c724 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 21:28:46 +0300 Subject: [PATCH 09/51] fix(storage): resolve the HDFS NameNode in relativize_path too An authority-less path now errors in relativize_path exactly as in create_operator unless hdfs.name-node is set, so the two cannot disagree. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 4 ++-- crates/storage/opendal/src/lib.rs | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 6e91e14e53..7567996cef 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -94,8 +94,8 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str /// Resolves the effective NameNode for a path โ€” the configured /// `hdfs.name-node` when set, else the path authority โ€” plus the relative -/// path. Both the operator cache and `delete_stream` batching key on this, -/// so they cannot drift apart. +/// path. The operator cache, `delete_stream` batching and `relativize_path` +/// all go through this, so they cannot drift apart. pub(crate) fn hdfs_native_effective_name_node<'a>( config: &HdfsNativeConfig, path: &'a str, diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 6b2f59595b..d9824bb513 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -500,8 +500,8 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::HdfsNative { .. } => { - let (_, relative_path) = hdfs_native_parse_path(path)?; + OpenDalStorage::HdfsNative { config, .. } => { + let (_, relative_path) = hdfs_native_effective_name_node(config, path)?; Ok(relative_path) } #[cfg(feature = "opendal-oss")] @@ -944,8 +944,17 @@ mod tests { #[cfg(feature = "opendal-hdfs-native")] #[test] fn test_relativize_path_hdfs_native_authority_less() { + use iceberg::io::HDFS_NAME_NODE; + + // Same rule as `create_operator`: usable only with `hdfs.name-node` set. let storage = hdfs_native_test_storage(); + assert!(storage.relativize_path("hdfs:///a/b").is_err()); + let props = HashMap::from([(HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string())]); + let storage = OpenDalStorage::HdfsNative { + config: Arc::new(hdfs_native_config_parse(props).unwrap()), + operators: HdfsNativeOperatorCache::default(), + }; assert_eq!(storage.relativize_path("hdfs:///a/b").unwrap(), "a/b"); } From 556a0f25f62c5b483445d8ed40a23d6efa2864c8 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 21:37:11 +0300 Subject: [PATCH 10/51] docs(storage): state the cost of the synchronous HDFS operator build Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 7567996cef..f36e335d6e 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -162,9 +162,10 @@ pub(crate) fn hdfs_native_create_operator<'a>( return Ok((op, relative_path)); } - // Built outside the lock: the build reads the Hadoop XML config - // synchronously. A racing first caller may build too; the loser is - // dropped before opening any connection. + // The build reads the Hadoop XML config synchronously (~0.1 ms, once + // per NameNode), so it runs outside the lock and is not worth a + // blocking-thread hop. A racing first caller may build too; the loser + // is dropped before opening any connection. let op = hdfs_native_operator_build(config, &name_node)?; Ok((operators.insert(name_node, op)?, relative_path)) } From d3ef1729afc80db98e5575d2aec1c9f5d94695c1 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 21:46:22 +0300 Subject: [PATCH 11/51] docs(storage): describe the HDFS operator cache lifecycle Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index f36e335d6e..d088c848d5 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -118,7 +118,8 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( /// Operators cached per effective NameNode: each holds an `hdfs-native` /// client with live RPC connections, whose tasks run on the tokio runtime -/// current when it was built. +/// current when it was built. The cache lives as long as the storage that +/// owns it (clones share it) and never evicts. #[derive(Clone, Debug, Default)] pub struct HdfsNativeOperatorCache(Arc>>); From 62ea36deb719c6f0d411f3502a6f3c9a941cd75d Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 21:53:21 +0300 Subject: [PATCH 12/51] fix(storage): normalize each entry of the hdfs.name-node list A space after a comma was passed through to opendal unchanged, which silently broke failover to that NameNode. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 35 ++++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index d088c848d5..d9fddb0b3b 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -32,11 +32,20 @@ use crate::utils::from_opendal_error; pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result { let mut cfg = HdfsNativeConfig::default(); - // `Operator::from_config` bypasses the builder's empty-string guard, and - // `Some("")` would shadow the path-authority fallback below. + // Entries are trimmed one by one: opendal splits the list on `,` as is, + // so a space after a comma would break failover to that NameNode. An + // empty result is dropped because `Operator::from_config` bypasses the + // builder's empty-string guard and `Some("")` would shadow the + // path-authority fallback below. if let Some(name_node) = m .remove(HDFS_NAME_NODE) - .map(|s| s.trim().trim_end_matches('/').to_string()) + .map(|s| { + s.split(',') + .map(|entry| entry.trim().trim_end_matches('/')) + .filter(|entry| !entry.is_empty()) + .collect::>() + .join(",") + }) .filter(|s| !s.is_empty()) { cfg.name_node = Some(name_node); @@ -243,11 +252,16 @@ mod tests { // Empty must not shadow the path-authority fallback. assert_eq!(parse(""), None); assert_eq!(parse(" "), None); - // Trailing `/` would otherwise yield a second cache entry for one cluster. + assert_eq!(parse(" , "), None); assert_eq!( parse(" hdfs://nn:8020/ ").as_deref(), Some("hdfs://nn:8020") ); + // Per entry: a space after the comma would break failover to nn2. + assert_eq!( + parse("hdfs://nn1:8020/, hdfs://nn2:8020/,").as_deref(), + Some("hdfs://nn1:8020,hdfs://nn2:8020") + ); } #[test] @@ -371,17 +385,24 @@ mod tests { fn test_hdfs_native_create_operator_configured_name_node_wins() { let config = hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), - "hdfs://configured:8020".to_string(), + "hdfs://nn1:8020/,hdfs://nn2:8020/".to_string(), )])) .unwrap(); let operators = HdfsNativeOperatorCache::default(); let (_, rel) = hdfs_native_create_operator("hdfs://from-path:9000/a/b", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs://nn1:8020/c", &config, &operators).unwrap(); + // The configured list is the single key, whatever authority the paths carry. assert_eq!(rel, "a/b"); - assert!(operators.get("hdfs://configured:8020").unwrap().is_some()); - assert!(operators.get("hdfs://from-path:9000").unwrap().is_none()); + assert_eq!(operators.len(), 1); + assert!( + operators + .get("hdfs://nn1:8020,hdfs://nn2:8020") + .unwrap() + .is_some() + ); } #[test] From 3ab457a65d4d0f2b9dd7cd308eb9ee96ec689987 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 22:13:29 +0300 Subject: [PATCH 13/51] feat(storage): accept PyIceberg's hdfs.host/hdfs.port for authority-less HDFS paths They map to Hadoop's fs.defaultFS and apply only when the path has no authority, as in PyIceberg; hdfs.name-node stays the explicit override. Co-Authored-By: Claude Fable 5.1 --- crates/iceberg/public-api.txt | 2 + crates/iceberg/src/io/storage/config/hdfs.rs | 20 ++- crates/storage/opendal/src/hdfs_native.rs | 151 +++++++++++++++++-- 3 files changed, 152 insertions(+), 21 deletions(-) diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index bf83f04dd6..a95235fbd8 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -1073,7 +1073,9 @@ pub const iceberg::io::GCS_SERVICE_HOST: &str pub const iceberg::io::GCS_TOKEN: &str pub const iceberg::io::GCS_USER_PROJECT: &str pub const iceberg::io::HDFS_HADOOP_CONF_PREFIX: &str +pub const iceberg::io::HDFS_HOST: &str pub const iceberg::io::HDFS_NAME_NODE: &str +pub const iceberg::io::HDFS_PORT: &str pub const iceberg::io::HF_ENDPOINT: &str pub const iceberg::io::HF_REVISION: &str pub const iceberg::io::HF_TOKEN: &str diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index 626f23ffb5..5531fc09e7 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -18,10 +18,20 @@ //! HDFS storage configuration. /// HDFS NameNode RPC endpoint(s), e.g. `hdfs://namenode:8020`; a -/// comma-separated list enables HA failover. When unset, the NameNode is -/// derived from the path authority. +/// comma-separated list enables HA failover. Takes precedence over the path +/// authority; when unset, the NameNode is derived from the path authority. pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; -/// Prefix for properties forwarded to the HDFS client configuration, e.g. -/// `hadoop.dfs.client.failover.random.order`. Forwarded values (prefix -/// stripped) override those loaded from `$HADOOP_CONF_DIR`. +/// NameNode host for authority-less paths, as in PyIceberg; paths that carry +/// an authority ignore it, as they do there. Combined with [`HDFS_PORT`] into +/// Hadoop's `fs.defaultFS`. PyIceberg's `hdfs.user` and `hdfs.kerberos_ticket` +/// have no equivalent: the client reads `HADOOP_USER_NAME` and the default +/// Kerberos credential cache. +pub const HDFS_HOST: &str = "hdfs.host"; +/// NameNode port for [`HDFS_HOST`]; defaults to `8020`. +pub const HDFS_PORT: &str = "hdfs.port"; +/// Prefix for properties forwarded to the HDFS client configuration with the +/// prefix stripped: `hadoop.dfs.client.failover.random.order` is forwarded as +/// `dfs.client.failover.random.order`. Forwarded values override those loaded +/// from `$HADOOP_CONF_DIR`; `hadoop.fs.defaultFS` also serves authority-less +/// paths. pub const HDFS_HADOOP_CONF_PREFIX: &str = "hadoop."; diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index d9fddb0b3b..2542168380 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_NAME_NODE}; +use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; use iceberg::{Error, ErrorKind, Result}; use opendal::Operator; use opendal::services::HdfsNativeConfig; @@ -28,6 +28,10 @@ use url::Url; use crate::utils::from_opendal_error; +/// Hadoop's default filesystem, which serves authority-less paths. +const FS_DEFAULT_FS: &str = "fs.defaultFS"; +const HDFS_DEFAULT_PORT: u16 = 8020; + /// Parse iceberg properties to [`HdfsNativeConfig`]. pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result { let mut cfg = HdfsNativeConfig::default(); @@ -51,13 +55,39 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result cfg.name_node = Some(name_node); } - let options: HashMap = m + let host = m.remove(HDFS_HOST).map(|s| s.trim().to_string()); + let port = m.remove(HDFS_PORT).map(|s| s.trim().to_string()); + + let mut options: HashMap = m .into_iter() .filter_map(|(key, value)| { key.strip_prefix(HDFS_HADOOP_CONF_PREFIX) .map(|stripped| (stripped.to_string(), value)) }) .collect(); + // PyIceberg's `hdfs.host`/`hdfs.port` name the filesystem for + // authority-less paths, which is what Hadoop's `fs.defaultFS` means; an + // explicit `hadoop.fs.defaultFS` wins. + if let Some(host) = host.filter(|s| !s.is_empty()) { + let port = match port.filter(|s| !s.is_empty()) { + Some(port) => port.parse::().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid `{HDFS_PORT}`: {port}: {e}"), + ) + })?, + None => HDFS_DEFAULT_PORT, + }; + // An IPv6 literal needs brackets in a URI authority. + let host = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]") + } else { + host + }; + options + .entry(FS_DEFAULT_FS.to_string()) + .or_insert_with(|| format!("hdfs://{host}:{port}")); + } if !options.is_empty() { cfg.options = Some(options); } @@ -102,9 +132,9 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str } /// Resolves the effective NameNode for a path โ€” the configured -/// `hdfs.name-node` when set, else the path authority โ€” plus the relative -/// path. The operator cache, `delete_stream` batching and `relativize_path` -/// all go through this, so they cannot drift apart. +/// `hdfs.name-node` when set, else the path authority, else `fs.defaultFS` โ€” +/// plus the relative path. The operator cache, `delete_stream` batching and +/// `relativize_path` all go through this, so they cannot drift apart. pub(crate) fn hdfs_native_effective_name_node<'a>( config: &HdfsNativeConfig, path: &'a str, @@ -114,17 +144,32 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( .name_node .clone() .or(authority_name_node) + .or_else(|| hdfs_native_default_fs(config)) .ok_or_else(|| { Error::new( ErrorKind::DataInvalid, format!( - "Invalid hdfs path: {path}, authority-less paths require the `{HDFS_NAME_NODE}` property" + "Invalid hdfs path: {path}, authority-less paths require `{HDFS_NAME_NODE}` or `{HDFS_HOST}`" ), ) })?; Ok((name_node, relative_path)) } +/// `fs.defaultFS` from the forwarded options, when it is an HDFS URI. +fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option { + config + .options + .as_ref()? + .get(FS_DEFAULT_FS) + .map(|s| s.trim().trim_end_matches('/')) + .filter(|s| { + s.strip_prefix("hdfs://") + .is_some_and(|rest| !rest.is_empty()) + }) + .map(str::to_string) +} + /// Operators cached per effective NameNode: each holds an `hdfs-native` /// client with live RPC connections, whose tasks run on the tokio runtime /// current when it was built. The cache lives as long as the storage that @@ -264,25 +309,99 @@ mod tests { ); } + #[test] + fn test_hdfs_native_config_parse_host_port_as_default_fs() { + let parse = |props: &[(&str, &str)]| { + let props = props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + hdfs_native_config_parse(props) + .map(|cfg| cfg.options.and_then(|o| o.get(FS_DEFAULT_FS).cloned())) + }; + + assert_eq!( + parse(&[(HDFS_HOST, "nn"), (HDFS_PORT, " 9000 ")]) + .unwrap() + .as_deref(), + Some("hdfs://nn:9000") + ); + assert_eq!( + parse(&[(HDFS_HOST, " nn ")]).unwrap().as_deref(), + Some("hdfs://nn:8020") + ); + assert_eq!( + parse(&[(HDFS_HOST, "::1")]).unwrap().as_deref(), + Some("hdfs://[::1]:8020") + ); + // An explicit `hadoop.fs.defaultFS` wins. + assert_eq!( + parse(&[ + (HDFS_HOST, "nn"), + ("hadoop.fs.defaultFS", "hdfs://explicit:8020") + ]) + .unwrap() + .as_deref(), + Some("hdfs://explicit:8020") + ); + // An empty host is unset; a port without a host has nothing to apply to. + assert_eq!( + parse(&[(HDFS_HOST, " "), (HDFS_PORT, "9000")]).unwrap(), + None + ); + assert_eq!(parse(&[(HDFS_PORT, "9000")]).unwrap(), None); + let err = parse(&[(HDFS_HOST, "nn"), (HDFS_PORT, "x")]).unwrap_err(); + assert!(err.to_string().contains(HDFS_PORT)); + } + + #[test] + fn test_hdfs_native_default_fs_must_be_hdfs() { + let parse = |default_fs: &str| { + hdfs_native_config_parse(HashMap::from([( + "hadoop.fs.defaultFS".to_string(), + default_fs.to_string(), + )])) + .unwrap() + }; + + let (nn, _) = + hdfs_native_effective_name_node(&parse("hdfs://nn:8020/"), "hdfs:///a").unwrap(); + assert_eq!(nn, "hdfs://nn:8020"); + for default_fs in ["viewfs://cluster/", "hdfs://", ""] { + assert!(hdfs_native_effective_name_node(&parse(default_fs), "hdfs:///a").is_err()); + } + } + #[test] fn test_hdfs_native_effective_name_node_precedence() { - let configured = hdfs_native_config_parse(HashMap::from([( - HDFS_NAME_NODE.to_string(), - "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), - )])) + let configured = hdfs_native_config_parse(HashMap::from([ + ( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), + ), + (HDFS_HOST.to_string(), "ignored".to_string()), + ])) .unwrap(); + let default_fs = + hdfs_native_config_parse(HashMap::from([(HDFS_HOST.to_string(), "nn".to_string())])) + .unwrap(); let unconfigured = HdfsNativeConfig::default(); - // Configured wins over the authority, including for authority-less paths. + // Configured wins over the authority and `hdfs.host`, including for + // authority-less paths. for path in ["hdfs://ns-a/x", "hdfs:///y"] { let (nn, _) = hdfs_native_effective_name_node(&configured, path).unwrap(); assert_eq!(nn, "hdfs://nn1:8020,hdfs://nn2:8020"); } - // Otherwise the authority, including its port. - let (nn, rel) = - hdfs_native_effective_name_node(&unconfigured, "hdfs://nn:9000/a/b").unwrap(); - assert_eq!((nn.as_str(), rel), ("hdfs://nn:9000", "a/b")); - // Neither: a pointed error. + // Otherwise the authority, including its port, even with `hdfs.host`. + for config in [&unconfigured, &default_fs] { + let (nn, rel) = hdfs_native_effective_name_node(config, "hdfs://nn:9000/a/b").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://nn:9000", "a/b")); + } + // Then `fs.defaultFS`, here from `hdfs.host`, for authority-less paths. + let (nn, rel) = hdfs_native_effective_name_node(&default_fs, "hdfs:///y").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://nn:8020", "y")); + // None of them: a pointed error. let err = hdfs_native_effective_name_node(&unconfigured, "hdfs:///a").unwrap_err(); assert!(err.to_string().contains(HDFS_NAME_NODE)); } From 74bb9b142ffe3e4f6a5761cc6c5b23a61456c736 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 22:37:45 +0300 Subject: [PATCH 14/51] fix(storage): require a port in every hdfs.name-node entry An entry without a port can only fail at the first I/O as 'invalid socket address'; reject it at build time with the property and entry named. Bare host:port stays accepted, as in Hadoop and opendal. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 63 ++++++++++++++++++----- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 2542168380..f9b47e6b03 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -41,18 +41,27 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result // empty result is dropped because `Operator::from_config` bypasses the // builder's empty-string guard and `Some("")` would shadow the // path-authority fallback below. - if let Some(name_node) = m - .remove(HDFS_NAME_NODE) - .map(|s| { - s.split(',') - .map(|entry| entry.trim().trim_end_matches('/')) - .filter(|entry| !entry.is_empty()) - .collect::>() - .join(",") - }) - .filter(|s| !s.is_empty()) - { - cfg.name_node = Some(name_node); + if let Some(name_node) = m.remove(HDFS_NAME_NODE) { + let entries: Vec<&str> = name_node + .split(',') + .map(|entry| entry.trim().trim_end_matches('/')) + .filter(|entry| !entry.is_empty()) + .collect(); + // hdfs-native dials each entry as `host:port` (`hdfs://` optional); + // without a port it fails only at the first I/O. + if let Some(entry) = entries.iter().find(|entry| { + entry + .rsplit_once(':') + .is_none_or(|(_, port)| port.parse::().is_err()) + }) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid `{HDFS_NAME_NODE}` entry: {entry}, expected host:port"), + )); + } + if !entries.is_empty() { + cfg.name_node = Some(entries.join(",")); + } } let host = m.remove(HDFS_HOST).map(|s| s.trim().to_string()); @@ -354,6 +363,36 @@ mod tests { assert!(err.to_string().contains(HDFS_PORT)); } + #[test] + fn test_hdfs_native_config_parse_name_node_requires_port() { + let parse = |value: &str| { + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + value.to_string(), + )])) + }; + + // `hdfs://` is optional: Hadoop's own rpc-address format is bare. + for value in [ + "nn:8020", + "hdfs://nn:8020", + "[::1]:8020", + "hdfs://nn1:8020,nn2:8020", + ] { + assert!(parse(value).is_ok(), "{value}"); + } + for value in [ + "hdfs://ns1", + "nn", + "nn:x", + "hdfs://[::1]", + "hdfs://nn1:8020,nn2", + ] { + let err = parse(value).unwrap_err().to_string(); + assert!(err.contains(HDFS_NAME_NODE), "{value}: {err}"); + } + } + #[test] fn test_hdfs_native_default_fs_must_be_hdfs() { let parse = |default_fs: &str| { From 0928265f0f58938794b2ab4aef1b80b1a1b034f4 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 22:55:53 +0300 Subject: [PATCH 15/51] docs(storage): state the runtime binding of cached HDFS operators Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index f9b47e6b03..ca77a3f174 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -181,8 +181,10 @@ fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option { /// Operators cached per effective NameNode: each holds an `hdfs-native` /// client with live RPC connections, whose tasks run on the tokio runtime -/// current when it was built. The cache lives as long as the storage that -/// owns it (clones share it) and never evicts. +/// current when it was built (a private one when built outside any). The +/// cache lives as long as the storage that owns it (clones share it) and +/// never evicts, so the storage must not be used from another runtime once +/// the building one is dropped: `hdfs-native` panics on a dead runtime. #[derive(Clone, Debug, Default)] pub struct HdfsNativeOperatorCache(Arc>>); @@ -539,6 +541,8 @@ mod tests { assert_eq!(hdfs_native_batch_key(&config, "hdfs:///a"), "hdfs:///a"); } + // The operator tests are plain `#[test]`s on purpose: building an + // operator must not need a runtime, as the siblings' tests also assume. #[test] fn test_hdfs_native_create_operator_configured_name_node_wins() { let config = hdfs_native_config_parse(HashMap::from([( From 089731f729bf8b53071c78bad989d0f271112985 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 23:08:26 +0300 Subject: [PATCH 16/51] test(storage): cover two-NameNode delete_stream batches and first-insert-wins caching Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 32 ++++++++++++++++ .../opendal/tests/file_io_hdfs_test.rs | 38 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index ca77a3f174..736704b526 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -591,6 +591,38 @@ mod tests { assert_eq!(operators.len(), 2); } + #[test] + fn test_hdfs_native_operator_cache_keeps_first_insert() { + let operators = HdfsNativeOperatorCache::default(); + let build = |root: &str| { + let mut config = hdfs_native_config_parse(HashMap::new()).unwrap(); + config.root = Some(root.to_string()); + hdfs_native_operator_build(&config, "hdfs://nn:8020").unwrap() + }; + + // Two callers racing for one NameNode: the first insert wins and both + // get the cached operator. + let first = operators + .insert("hdfs://nn:8020".to_string(), build("/first")) + .unwrap(); + let second = operators + .insert("hdfs://nn:8020".to_string(), build("/second")) + .unwrap(); + assert_eq!(first.info().root(), "/first/"); + assert_eq!(second.info().root(), "/first/"); + assert_eq!(operators.len(), 1); + + let config = HdfsNativeConfig::default(); + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + hdfs_native_create_operator("hdfs://other:8020/a", &config, &operators).unwrap() + }); + } + }); + assert_eq!(operators.len(), 2); + } + #[test] fn test_hdfs_native_create_operator_authority_less_without_config_errors() { let config = HdfsNativeConfig::default(); diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index 816c784189..907151cafb 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -177,6 +177,44 @@ mod tests { } } + /// Paths are batched per effective NameNode; two spellings of the fixture's + /// NameNode drive two batches through one stream. The fixture is one + /// cluster, so the keying itself is pinned by the unit tests. + #[tokio::test] + async fn test_file_io_hdfs_delete_stream_two_name_nodes() { + require_hdfs!(); + let file_io = get_file_io(); + let dir = test_path("test_file_io_hdfs_delete_stream_two_name_nodes"); + let alt_dir = dir.replacen("localhost", "127.0.0.1", 1); + + let paths = vec![format!("{dir}/a"), format!("{alt_dir}/b")]; + for path in &paths { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write("delete-me".into()) + .await + .unwrap(); + } + // Both spellings reach the same cluster. + assert!(file_io.exists(&format!("{alt_dir}/a")).await.unwrap()); + assert!(file_io.exists(&format!("{dir}/b")).await.unwrap()); + + let stream = futures::stream::iter(paths.clone()).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + // Both batches ran. + for path in [ + format!("{dir}/a"), + format!("{dir}/b"), + format!("{alt_dir}/a"), + format!("{alt_dir}/b"), + ] { + assert!(!file_io.exists(&path).await.unwrap(), "{path}"); + } + } + #[tokio::test] async fn test_file_io_hdfs_delete_stream_empty() { require_hdfs!(); From 22db8e6f6c3be85fbc06186365862ef40ba24075 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 3 Oct 2026 23:09:02 +0300 Subject: [PATCH 17/51] test(utils): name the HDFS port constant like its siblings Co-Authored-By: Claude Fable 5.1 --- crates/test_utils/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/test_utils/src/lib.rs b/crates/test_utils/src/lib.rs index ae82b54ad3..176767813e 100644 --- a/crates/test_utils/src/lib.rs +++ b/crates/test_utils/src/lib.rs @@ -55,7 +55,7 @@ mod common { pub const DEFAULT_HMS_PORT: u16 = 9083; pub const DEFAULT_GLUE_PORT: u16 = 5001; pub const DEFAULT_GCS_PORT: u16 = 4443; - pub const DEFAULT_HDFS_NN_PORT: u16 = 8020; + pub const DEFAULT_HDFS_PORT: u16 = 8020; /// Returns the S3-compatible object store endpoint. /// Checks ICEBERG_TEST_OBJECT_STORE_ENDPOINT env var, otherwise returns localhost default. @@ -95,7 +95,7 @@ mod common { /// Checks ICEBERG_TEST_HDFS_ENDPOINT env var, otherwise returns localhost default. pub fn get_hdfs_endpoint() -> String { std::env::var(ENV_HDFS_ENDPOINT) - .unwrap_or_else(|_| format!("hdfs://localhost:{DEFAULT_HDFS_NN_PORT}")) + .unwrap_or_else(|_| format!("hdfs://localhost:{DEFAULT_HDFS_PORT}")) } /// Helper to clean up a namespace and its tables before a test runs. From 629cfc8f33a00ee932c5c4cc4bb2cd077c089310 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 12:07:18 +0300 Subject: [PATCH 18/51] fix(storage): rebuild cached HDFS operators once their runtime is gone Each cache entry parks a sentinel task on the runtime that built the operator and keeps a Weak to a token the task owns. When that runtime shuts down the token dies, the entry reads as stale and the operator is rebuilt on the current runtime instead of hdfs-native panicking on the dead one. Sentinels are aborted when entries drop. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/Cargo.toml | 3 +- crates/storage/opendal/src/hdfs_native.rs | 156 ++++++++++++++++-- .../opendal/tests/hdfs_native_runtime_test.rs | 78 +++++++++ 3 files changed, 221 insertions(+), 16 deletions(-) create mode 100644 crates/storage/opendal/tests/hdfs_native_runtime_test.rs diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index a7591606b7..4f5f2e29e3 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -42,7 +42,7 @@ opendal-all = [ opendal-azdls = ["opendal/services-azdls"] opendal-fs = ["opendal/services-fs"] opendal-gcs = ["opendal/services-gcs"] -opendal-hdfs-native = ["opendal/services-hdfs-native"] +opendal-hdfs-native = ["opendal/services-hdfs-native", "dep:tokio"] opendal-hf = ["opendal/services-hf"] opendal-memory = ["opendal/services-memory"] opendal-oss = ["opendal/services-oss"] @@ -60,6 +60,7 @@ opendal = { workspace = true } reqsign-aws-v4 = { version = "3.0.0", optional = true } reqsign-core = { version = "3.0.0", optional = true } serde = { workspace = true } +tokio = { workspace = true, optional = true } typetag = { workspace = true } url = { workspace = true } diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 736704b526..36cf87c9a5 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -18,12 +18,15 @@ //! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). use std::collections::HashMap; -use std::sync::{Arc, RwLock}; +use std::collections::hash_map::Entry; +use std::sync::{Arc, RwLock, Weak}; use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; use iceberg::{Error, ErrorKind, Result}; use opendal::Operator; use opendal::services::HdfsNativeConfig; +use tokio::runtime::Handle; +use tokio::task::JoinHandle; use url::Url; use crate::utils::from_opendal_error; @@ -181,28 +184,90 @@ fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option { /// Operators cached per effective NameNode: each holds an `hdfs-native` /// client with live RPC connections, whose tasks run on the tokio runtime -/// current when it was built (a private one when built outside any). The -/// cache lives as long as the storage that owns it (clones share it) and -/// never evicts, so the storage must not be used from another runtime once -/// the building one is dropped: `hdfs-native` panics on a dead runtime. +/// current when it was built (a private one when built outside any). An +/// entry is rebuilt once that runtime is gone, as `hdfs-native` panics when +/// it spawns onto a dead one. The cache lives as long as the storage that +/// owns it (clones share it). #[derive(Clone, Debug, Default)] -pub struct HdfsNativeOperatorCache(Arc>>); +pub struct HdfsNativeOperatorCache(Arc>>); + +#[derive(Debug)] +struct CachedOperator { + operator: Operator, + /// `None` when built outside any runtime. + sentinel: Option, +} + +/// A task parked on the building runtime that owns the token, so the token +/// outlives it only while that runtime is alive. Aborted on drop so entries +/// do not leave parked tasks behind. +#[derive(Debug)] +struct RuntimeSentinel { + alive: Weak<()>, + task: JoinHandle<()>, +} + +impl RuntimeSentinel { + fn spawn(handle: &Handle) -> Self { + let token = Arc::new(()); + let alive = Arc::downgrade(&token); + let task = handle.spawn(async move { + let _token = token; + std::future::pending::<()>().await + }); + Self { alive, task } + } +} + +impl Drop for RuntimeSentinel { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl CachedOperator { + fn new(operator: Operator) -> Self { + let sentinel = Handle::try_current() + .ok() + .map(|handle| RuntimeSentinel::spawn(&handle)); + Self { operator, sentinel } + } + + fn runtime_alive(&self) -> bool { + self.sentinel + .as_ref() + .is_none_or(|sentinel| sentinel.alive.strong_count() > 0) + } +} impl HdfsNativeOperatorCache { fn get(&self, name_node: &str) -> Result> { - Ok(self.0.read().map_err(poisoned)?.get(name_node).cloned()) + Ok(self + .0 + .read() + .map_err(poisoned)? + .get(name_node) + .filter(|cached| cached.runtime_alive()) + .map(|cached| cached.operator.clone())) } /// Inserts `op` unless a concurrent caller got there first, returning - /// whichever operator the cache now holds. + /// whichever operator the cache now holds; a stale entry is replaced. fn insert(&self, name_node: String, op: Operator) -> Result { - Ok(self - .0 - .write() - .map_err(poisoned)? - .entry(name_node) - .or_insert(op) - .clone()) + let mut operators = self.0.write().map_err(poisoned)?; + match operators.entry(name_node) { + Entry::Occupied(entry) if entry.get().runtime_alive() => { + Ok(entry.get().operator.clone()) + } + Entry::Occupied(mut entry) => { + entry.insert(CachedOperator::new(op.clone())); + Ok(op) + } + Entry::Vacant(entry) => { + entry.insert(CachedOperator::new(op.clone())); + Ok(op) + } + } } #[cfg(test)] @@ -623,6 +688,67 @@ mod tests { assert_eq!(operators.len(), 2); } + #[test] + fn test_hdfs_native_operator_cache_rebuilds_after_runtime_shutdown() { + let flavors: [fn() -> tokio::runtime::Runtime; 2] = [ + || { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + }, + || tokio::runtime::Builder::new_multi_thread().build().unwrap(), + ]; + for build_runtime in flavors { + let operators = HdfsNativeOperatorCache::default(); + let config = HdfsNativeConfig::default(); + + let runtime = build_runtime(); + runtime.block_on(async { + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); + }); + assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); + + // The building runtime is gone: the entry is stale and gets rebuilt. + drop(runtime); + assert!(operators.get("hdfs://nn:8020").unwrap().is_none()); + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); + assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); + assert_eq!(operators.len(), 1); + } + } + + #[test] + fn test_hdfs_native_operator_cache_drops_its_sentinel_task() { + let flavors: [fn() -> tokio::runtime::Runtime; 2] = [ + || { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + }, + || tokio::runtime::Builder::new_multi_thread().build().unwrap(), + ]; + for build_runtime in flavors { + build_runtime().block_on(async { + let metrics = Handle::current().metrics(); + let baseline = metrics.num_alive_tasks(); + + let operators = HdfsNativeOperatorCache::default(); + let config = HdfsNativeConfig::default(); + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); + assert_eq!(metrics.num_alive_tasks(), baseline + 1); + + // Dropping the cache aborts the sentinel; the abort lands once + // the runtime schedules the task. + drop(operators); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while metrics.num_alive_tasks() != baseline { + assert!(std::time::Instant::now() < deadline, "sentinel task leaked"); + tokio::task::yield_now().await; + } + }); + } + } + #[test] fn test_hdfs_native_create_operator_authority_less_without_config_errors() { let config = HdfsNativeConfig::default(); diff --git a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs new file mode 100644 index 0000000000..2d9cb205f2 --- /dev/null +++ b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A cached HDFS operator is bound to the tokio runtime that built it; once +//! that runtime is gone it must be rebuilt rather than reused. Needs no +//! HDFS: the NameNode is a local listener that only counts dials. + +#[cfg(feature = "opendal-hdfs-native")] +mod tests { + use std::net::TcpListener; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use iceberg::io::{FileIO, FileIOBuilder, HDFS_NAME_NODE}; + use iceberg_storage_opendal::OpenDalStorageFactory; + + /// Accepts and immediately closes connections, counting them. + fn fake_name_node() -> (u16, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let dials = Arc::new(AtomicUsize::new(0)); + let counter = dials.clone(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(_stream) = stream else { break }; + counter.fetch_add(1, Ordering::SeqCst); + } + }); + (port, dials) + } + + fn stat(runtime: &tokio::runtime::Runtime, file_io: &FileIO) { + runtime.block_on(async { + let input = file_io.new_input("hdfs://x/f").unwrap(); + // The fake NameNode never answers, so this fails; only the dial matters. + let _ = tokio::time::timeout(Duration::from_secs(10), input.metadata()).await; + }); + } + + #[test] + fn test_hdfs_operator_is_rebuilt_after_its_runtime_is_dropped() { + let (port, dials) = fake_name_node(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) + .with_prop(HDFS_NAME_NODE, format!("hdfs://127.0.0.1:{port}")) + .with_prop("hadoop.dfs.client.failover.max.attempts", "1") + .build(); + + let first = tokio::runtime::Runtime::new().unwrap(); + stat(&first, &file_io); + let after_first = dials.load(Ordering::SeqCst); + assert!(after_first >= 1, "the NameNode was never dialed"); + drop(first); + + // Reusing the FileIO from another runtime used to panic inside + // hdfs-native, whose client was bound to the dropped runtime. + let second = tokio::runtime::Runtime::new().unwrap(); + stat(&second, &file_io); + assert!( + dials.load(Ordering::SeqCst) > after_first, + "no dial from the second runtime" + ); + } +} From e38bab2f047ab6d021b99299a6f47e861ad02804 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 12:54:04 +0300 Subject: [PATCH 19/51] fix(storage): resolve HDFS NameNodes like Hadoop does An authority with a port is used as is; hdfs.name-node serves logical nameservice authorities and authority-less paths, which otherwise fall back to fs.defaultFS. Portless strings from any source are logical names that only hdfs.name-node resolves, reported at resolution instead of at the first I/O. Co-Authored-By: Claude Fable 5.1 --- crates/iceberg/src/io/storage/config/hdfs.rs | 7 +- crates/storage/opendal/src/hdfs_native.rs | 120 ++++++++++++------- crates/storage/opendal/src/lib.rs | 26 +++- 3 files changed, 102 insertions(+), 51 deletions(-) diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index 5531fc09e7..a363965829 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -17,9 +17,10 @@ //! HDFS storage configuration. -/// HDFS NameNode RPC endpoint(s), e.g. `hdfs://namenode:8020`; a -/// comma-separated list enables HA failover. Takes precedence over the path -/// authority; when unset, the NameNode is derived from the path authority. +/// HDFS NameNode RPC endpoint(s) as `host:port`, e.g. `hdfs://namenode:8020`; +/// a comma-separated list enables HA failover. Resolves logical nameservice +/// authorities (no port) and authority-less paths; as in Hadoop, an authority +/// with a port is always used as is. pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; /// NameNode host for authority-less paths, as in PyIceberg; paths that carry /// an authority ignore it, as they do there. Combined with [`HDFS_PORT`] into diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 36cf87c9a5..cc86f6816c 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -35,6 +35,14 @@ use crate::utils::from_opendal_error; const FS_DEFAULT_FS: &str = "fs.defaultFS"; const HDFS_DEFAULT_PORT: u16 = 8020; +/// `hdfs-native` dials a NameNode as a socket address and has no default +/// port, so anything without one can only be a logical nameservice name. +fn hdfs_native_has_port(name_node: &str) -> bool { + name_node + .rsplit_once(':') + .is_some_and(|(_, port)| port.parse::().is_ok()) +} + /// Parse iceberg properties to [`HdfsNativeConfig`]. pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result { let mut cfg = HdfsNativeConfig::default(); @@ -50,13 +58,9 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result .map(|entry| entry.trim().trim_end_matches('/')) .filter(|entry| !entry.is_empty()) .collect(); - // hdfs-native dials each entry as `host:port` (`hdfs://` optional); - // without a port it fails only at the first I/O. - if let Some(entry) = entries.iter().find(|entry| { - entry - .rsplit_once(':') - .is_none_or(|(_, port)| port.parse::().is_err()) - }) { + // Each entry is dialed as `host:port` (`hdfs://` optional); a portless + // one would fail only at the first I/O. + if let Some(entry) = entries.iter().find(|entry| !hdfs_native_has_port(entry)) { return Err(Error::new( ErrorKind::DataInvalid, format!("Invalid `{HDFS_NAME_NODE}` entry: {entry}, expected host:port"), @@ -143,28 +147,49 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str Ok((name_node, rel)) } -/// Resolves the effective NameNode for a path โ€” the configured -/// `hdfs.name-node` when set, else the path authority, else `fs.defaultFS` โ€” -/// plus the relative path. The operator cache, `delete_stream` batching and -/// `relativize_path` all go through this, so they cannot drift apart. +/// Resolves the effective NameNode for a path, plus the relative path. As in +/// Hadoop, an authority with a port is used as is; a logical nameservice +/// authority (no port) resolves through `hdfs.name-node`, and an +/// authority-less path through `hdfs.name-node`, else `fs.defaultFS`. The +/// operator cache, `delete_stream` batching and `relativize_path` all go +/// through this, so they cannot drift apart. pub(crate) fn hdfs_native_effective_name_node<'a>( config: &HdfsNativeConfig, path: &'a str, ) -> Result<(String, &'a str)> { - let (authority_name_node, relative_path) = hdfs_native_parse_path(path)?; - let name_node = config - .name_node - .clone() - .or(authority_name_node) - .or_else(|| hdfs_native_default_fs(config)) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid hdfs path: {path}, authority-less paths require `{HDFS_NAME_NODE}` or `{HDFS_HOST}`" - ), - ) - })?; + let (authority, relative_path) = hdfs_native_parse_path(path)?; + let invalid = |reason: String| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path: {path}, {reason}"), + ) + }; + let name_node = match authority { + Some(authority) if hdfs_native_has_port(&authority) => authority, + Some(logical) => config.name_node.clone().ok_or_else(|| { + let logical = logical.trim_start_matches("hdfs://"); + invalid(format!( + "logical nameservice `{logical}` requires `{HDFS_NAME_NODE}`" + )) + })?, + None => match config + .name_node + .clone() + .or_else(|| hdfs_native_default_fs(config)) + { + Some(name_node) if hdfs_native_has_port(&name_node) => name_node, + Some(logical) => { + return Err(invalid(format!( + "`{FS_DEFAULT_FS}` {logical} has no port, a logical nameservice requires `{HDFS_NAME_NODE}`" + ))); + } + None => { + return Err(invalid(format!( + "authority-less paths require `{HDFS_NAME_NODE}` or `{HDFS_HOST}`" + ))); + } + }, + }; Ok((name_node, relative_path)) } @@ -476,6 +501,9 @@ mod tests { for default_fs in ["viewfs://cluster/", "hdfs://", ""] { assert!(hdfs_native_effective_name_node(&parse(default_fs), "hdfs:///a").is_err()); } + // Portless: a logical nameservice, which only `hdfs.name-node` resolves. + let err = hdfs_native_effective_name_node(&parse("hdfs://ns1"), "hdfs:///a").unwrap_err(); + assert!(err.to_string().contains("has no port"), "{err}"); } #[test] @@ -493,23 +521,28 @@ mod tests { .unwrap(); let unconfigured = HdfsNativeConfig::default(); - // Configured wins over the authority and `hdfs.host`, including for - // authority-less paths. + // An authority with a port is used as is, whatever is configured. + for config in [&configured, &default_fs, &unconfigured] { + let (nn, rel) = + hdfs_native_effective_name_node(config, "hdfs://other:9000/a/b").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://other:9000", "a/b")); + } + // A logical nameservice and an authority-less path use the configured + // list, ahead of `hdfs.host`. for path in ["hdfs://ns-a/x", "hdfs:///y"] { let (nn, _) = hdfs_native_effective_name_node(&configured, path).unwrap(); assert_eq!(nn, "hdfs://nn1:8020,hdfs://nn2:8020"); } - // Otherwise the authority, including its port, even with `hdfs.host`. - for config in [&unconfigured, &default_fs] { - let (nn, rel) = hdfs_native_effective_name_node(config, "hdfs://nn:9000/a/b").unwrap(); - assert_eq!((nn.as_str(), rel), ("hdfs://nn:9000", "a/b")); - } - // Then `fs.defaultFS`, here from `hdfs.host`, for authority-less paths. + // Only an authority-less path falls back to `fs.defaultFS`. let (nn, rel) = hdfs_native_effective_name_node(&default_fs, "hdfs:///y").unwrap(); assert_eq!((nn.as_str(), rel), ("hdfs://nn:8020", "y")); - // None of them: a pointed error. - let err = hdfs_native_effective_name_node(&unconfigured, "hdfs:///a").unwrap_err(); - assert!(err.to_string().contains(HDFS_NAME_NODE)); + let err = hdfs_native_effective_name_node(&default_fs, "hdfs://ns-a/x").unwrap_err(); + assert!(err.to_string().contains("logical nameservice `ns-a`")); + // Nothing applicable: pointed errors. + for path in ["hdfs:///a", "hdfs://ns-a/x"] { + let err = hdfs_native_effective_name_node(&unconfigured, path).unwrap_err(); + assert!(err.to_string().contains(HDFS_NAME_NODE), "{err}"); + } } #[test] @@ -609,7 +642,7 @@ mod tests { // The operator tests are plain `#[test]`s on purpose: building an // operator must not need a runtime, as the siblings' tests also assume. #[test] - fn test_hdfs_native_create_operator_configured_name_node_wins() { + fn test_hdfs_native_create_operator_configured_name_node_serves_logical_paths() { let config = hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), "hdfs://nn1:8020/,hdfs://nn2:8020/".to_string(), @@ -617,11 +650,9 @@ mod tests { .unwrap(); let operators = HdfsNativeOperatorCache::default(); - let (_, rel) = - hdfs_native_create_operator("hdfs://from-path:9000/a/b", &config, &operators).unwrap(); - hdfs_native_create_operator("hdfs://nn1:8020/c", &config, &operators).unwrap(); - - // The configured list is the single key, whatever authority the paths carry. + // Logical and authority-less paths share the configured list's operator. + let (_, rel) = hdfs_native_create_operator("hdfs://ns-a/a/b", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs:///c", &config, &operators).unwrap(); assert_eq!(rel, "a/b"); assert_eq!(operators.len(), 1); assert!( @@ -630,6 +661,11 @@ mod tests { .unwrap() .is_some() ); + + // A concrete authority is another cluster, never the configured one. + hdfs_native_create_operator("hdfs://other:9000/d", &config, &operators).unwrap(); + assert_eq!(operators.len(), 2); + assert!(operators.get("hdfs://other:9000").unwrap().is_some()); } #[test] diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index bd236bcc3f..dda18ca56b 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1224,19 +1224,33 @@ mod tests { #[cfg(feature = "opendal-hdfs-native")] #[test] fn test_relativize_path_hdfs_native() { - let storage = hdfs_native_test_storage(); + use iceberg::io::HDFS_NAME_NODE; + let storage = hdfs_native_test_storage(); assert_eq!( storage - .relativize_path("hdfs://nameservice1/a/b.parquet") + .relativize_path("hdfs://nn:8020/warehouse/db/t") .unwrap(), - "a/b.parquet" + "warehouse/db/t" ); - assert_eq!( + + // A logical nameservice resolves only through `hdfs.name-node`. + assert!( storage - .relativize_path("hdfs://nn:8020/warehouse/db/t") + .relativize_path("hdfs://nameservice1/a/b.parquet") + .is_err() + ); + let props = HashMap::from([(HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string())]); + let configured = OpenDalStorage::HdfsNative { + config: Arc::new(hdfs_native_config_parse(props).unwrap()), + operators: HdfsNativeOperatorCache::default(), + client_config: OpenDalClientConfig::default(), + }; + assert_eq!( + configured + .relativize_path("hdfs://nameservice1/a/b.parquet") .unwrap(), - "warehouse/db/t" + "a/b.parquet" ); } From cfe4ee783779fe6678f76f32f8314d72cdaebc17 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 13:16:01 +0300 Subject: [PATCH 20/51] refactor(storage): keep the HDFS operator cache out of the public API OpenDalStorage::HdfsNative now wraps one opaque HdfsNativeStorage with crate-private fields, built only by the factories. The serialized form keeps its nesting; the io-timeout propagation test now covers the HDFS factory too. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/public-api.txt | 21 ++--- crates/storage/opendal/src/hdfs_native.rs | 28 +++++- crates/storage/opendal/src/lib.rs | 110 +++++++++++++--------- crates/storage/opendal/src/resolving.rs | 9 +- 4 files changed, 107 insertions(+), 61 deletions(-) diff --git a/crates/storage/opendal/public-api.txt b/crates/storage/opendal/public-api.txt index bd6ca9c9a6..797d90d1e8 100644 --- a/crates/storage/opendal/public-api.txt +++ b/crates/storage/opendal/public-api.txt @@ -8,10 +8,7 @@ pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::HdfsNative -pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::client_config: iceberg_storage_opendal::OpenDalClientConfig -pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::config: alloc::sync::Arc -pub iceberg_storage_opendal::OpenDalStorage::HdfsNative::operators: iceberg_storage_opendal::HdfsNativeOperatorCache +pub iceberg_storage_opendal::OpenDalStorage::HdfsNative(iceberg_storage_opendal::HdfsNativeStorage) pub iceberg_storage_opendal::OpenDalStorage::Hf pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc @@ -74,13 +71,15 @@ impl core::clone::Clone for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::clone(&self) -> Self impl core::fmt::Debug for iceberg_storage_opendal::CustomAwsCredentialLoader pub fn iceberg_storage_opendal::CustomAwsCredentialLoader::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -pub struct iceberg_storage_opendal::HdfsNativeOperatorCache(_) -impl core::clone::Clone for iceberg_storage_opendal::HdfsNativeOperatorCache -pub fn iceberg_storage_opendal::HdfsNativeOperatorCache::clone(&self) -> iceberg_storage_opendal::HdfsNativeOperatorCache -impl core::default::Default for iceberg_storage_opendal::HdfsNativeOperatorCache -pub fn iceberg_storage_opendal::HdfsNativeOperatorCache::default() -> iceberg_storage_opendal::HdfsNativeOperatorCache -impl core::fmt::Debug for iceberg_storage_opendal::HdfsNativeOperatorCache -pub fn iceberg_storage_opendal::HdfsNativeOperatorCache::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct iceberg_storage_opendal::HdfsNativeStorage +impl core::clone::Clone for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::clone(&self) -> iceberg_storage_opendal::HdfsNativeStorage +impl core::fmt::Debug for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl serde_core::ser::Serialize for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::HdfsNativeStorage +pub fn iceberg_storage_opendal::HdfsNativeStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg_storage_opendal::OpenDalClientConfig impl iceberg_storage_opendal::OpenDalClientConfig pub fn iceberg_storage_opendal::OpenDalClientConfig::from_properties(properties: &std::collections::hash::map::HashMap) -> iceberg::error::Result diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index cc86f6816c..40682aa566 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -25,10 +25,12 @@ use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT} use iceberg::{Error, ErrorKind, Result}; use opendal::Operator; use opendal::services::HdfsNativeConfig; +use serde::{Deserialize, Serialize}; use tokio::runtime::Handle; use tokio::task::JoinHandle; use url::Url; +use crate::OpenDalClientConfig; use crate::utils::from_opendal_error; /// Hadoop's default filesystem, which serves authority-less paths. @@ -207,6 +209,28 @@ fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option { .map(str::to_string) } +/// State of [`OpenDalStorage::HdfsNative`](crate::OpenDalStorage::HdfsNative): +/// the parsed configuration and the per-NameNode operator cache. Only the +/// storage factories build it. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct HdfsNativeStorage { + pub(crate) config: Arc, + #[serde(skip, default)] + pub(crate) operators: HdfsNativeOperatorCache, + #[serde(default)] + pub(crate) client_config: OpenDalClientConfig, +} + +impl HdfsNativeStorage { + pub(crate) fn new(config: HdfsNativeConfig, client_config: OpenDalClientConfig) -> Self { + Self { + config: Arc::new(config), + operators: HdfsNativeOperatorCache::default(), + client_config, + } + } +} + /// Operators cached per effective NameNode: each holds an `hdfs-native` /// client with live RPC connections, whose tasks run on the tokio runtime /// current when it was built (a private one when built outside any). An @@ -214,7 +238,7 @@ fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option { /// it spawns onto a dead one. The cache lives as long as the storage that /// owns it (clones share it). #[derive(Clone, Debug, Default)] -pub struct HdfsNativeOperatorCache(Arc>>); +pub(crate) struct HdfsNativeOperatorCache(Arc>>); #[derive(Debug)] struct CachedOperator { @@ -266,7 +290,7 @@ impl CachedOperator { } impl HdfsNativeOperatorCache { - fn get(&self, name_node: &str) -> Result> { + pub(crate) fn get(&self, name_node: &str) -> Result> { Ok(self .0 .read() diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index dda18ca56b..ddc207e136 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -81,8 +81,7 @@ cfg_if! { if #[cfg(feature = "opendal-hdfs-native")] { mod hdfs_native; use hdfs_native::*; - pub use hdfs_native::HdfsNativeOperatorCache; - use opendal::services::HdfsNativeConfig; + pub use hdfs_native::HdfsNativeStorage; } } @@ -249,11 +248,12 @@ impl StorageFactory for OpenDalStorageFactory { client_config, })), #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative { - config: hdfs_native_config_parse(config.props().clone())?.into(), - operators: HdfsNativeOperatorCache::default(), - client_config, - })), + OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative( + HdfsNativeStorage::new( + hdfs_native_config_parse(config.props().clone())?, + client_config, + ), + ))), #[cfg(feature = "opendal-oss")] OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss { config: oss_config_parse(config.props().clone())?.into(), @@ -339,21 +339,9 @@ pub enum OpenDalStorage { #[serde(default)] client_config: OpenDalClientConfig, }, - /// HDFS storage variant. - /// - /// The NameNode is taken from the `hdfs.name-node` property when set - /// (comma-separated endpoints enable HA failover), else the path authority. + /// HDFS storage variant; see [`HdfsNativeStorage`]. #[cfg(feature = "opendal-hdfs-native")] - HdfsNative { - /// HDFS configuration. - config: Arc, - /// Operator cache keyed by effective NameNode. - #[serde(skip, default)] - operators: HdfsNativeOperatorCache, - /// Backend-independent client settings. - #[serde(default)] - client_config: OpenDalClientConfig, - }, + HdfsNative(HdfsNativeStorage), /// OSS storage variant. #[cfg(feature = "opendal-oss")] Oss { @@ -470,9 +458,9 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::HdfsNative { - config, operators, .. - } => hdfs_native_create_operator(path, config, operators)?, + OpenDalStorage::HdfsNative(storage) => { + hdfs_native_create_operator(path, &storage.config, &storage.operators)? + } #[cfg(feature = "opendal-oss")] OpenDalStorage::Oss { config, .. } => { let op = oss_config_build(config, path)?; @@ -540,7 +528,7 @@ impl OpenDalStorage { #[cfg(feature = "opendal-hf")] OpenDalStorage::Hf { client_config, .. } => client_config, #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::HdfsNative { client_config, .. } => client_config, + OpenDalStorage::HdfsNative(storage) => &storage.client_config, #[cfg(all( not(feature = "opendal-memory"), not(feature = "opendal-s3"), @@ -566,7 +554,7 @@ impl OpenDalStorage { // The URL host alone would merge distinct NameNodes that differ // only by port; key by the effective NameNode instead. #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::HdfsNative { config, .. } => hdfs_native_batch_key(config, path), + OpenDalStorage::HdfsNative(storage) => hdfs_native_batch_key(&storage.config, path), _ => url::Url::parse(path) .ok() .and_then(|u| u.host_str().map(|s| s.to_string())) @@ -627,8 +615,8 @@ impl OpenDalStorage { } } #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorage::HdfsNative { config, .. } => { - let (_, relative_path) = hdfs_native_effective_name_node(config, path)?; + OpenDalStorage::HdfsNative(storage) => { + let (_, relative_path) = hdfs_native_effective_name_node(&storage.config, path)?; Ok(relative_path) } #[cfg(feature = "opendal-oss")] @@ -982,6 +970,8 @@ mod tests { OpenDalStorageFactory::Azdls, #[cfg(feature = "opendal-hf")] OpenDalStorageFactory::Hf, + #[cfg(feature = "opendal-hdfs-native")] + OpenDalStorageFactory::HdfsNative, ]; for factory in factories { // `build` returns `dyn Storage`, so read the config back from its serialized form. @@ -1214,11 +1204,47 @@ mod tests { #[cfg(feature = "opendal-hdfs-native")] fn hdfs_native_test_storage() -> OpenDalStorage { - OpenDalStorage::HdfsNative { - config: Arc::new(HdfsNativeConfig::default()), - operators: HdfsNativeOperatorCache::default(), - client_config: OpenDalClientConfig::default(), - } + OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + opendal::services::HdfsNativeConfig::default(), + OpenDalClientConfig::default(), + )) + } + + /// The configuration round-trips through serde; the operator cache does + /// not and starts empty. + #[cfg(feature = "opendal-hdfs-native")] + #[test] + fn test_hdfs_native_storage_serde_round_trip() { + use iceberg::io::HDFS_NAME_NODE; + + let props = HashMap::from([ + (HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string()), + ( + "hadoop.dfs.client.use.datanode.hostname".to_string(), + "true".to_string(), + ), + ]); + let storage = OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + client_config("45000").unwrap(), + )); + + let value = serde_json::to_value(&storage).unwrap(); + assert!(value["HdfsNative"].get("operators").is_none()); + let OpenDalStorage::HdfsNative(restored) = serde_json::from_value(value).unwrap() else { + panic!("expected the HdfsNative variant"); + }; + assert_eq!(restored.config.name_node.as_deref(), Some("hdfs://nn:8020")); + assert_eq!( + restored + .config + .options + .as_ref() + .and_then(|o| o.get("dfs.client.use.datanode.hostname")), + Some(&"true".to_string()) + ); + assert_eq!(restored.client_config.io_timeout(), Duration::from_secs(45)); + assert!(restored.operators.get("hdfs://nn:8020").unwrap().is_none()); } #[cfg(feature = "opendal-hdfs-native")] @@ -1241,11 +1267,10 @@ mod tests { .is_err() ); let props = HashMap::from([(HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string())]); - let configured = OpenDalStorage::HdfsNative { - config: Arc::new(hdfs_native_config_parse(props).unwrap()), - operators: HdfsNativeOperatorCache::default(), - client_config: OpenDalClientConfig::default(), - }; + let configured = OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + OpenDalClientConfig::default(), + )); assert_eq!( configured .relativize_path("hdfs://nameservice1/a/b.parquet") @@ -1264,11 +1289,10 @@ mod tests { assert!(storage.relativize_path("hdfs:///a/b").is_err()); let props = HashMap::from([(HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string())]); - let storage = OpenDalStorage::HdfsNative { - config: Arc::new(hdfs_native_config_parse(props).unwrap()), - operators: HdfsNativeOperatorCache::default(), - client_config: OpenDalClientConfig::default(), - }; + let storage = OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + OpenDalClientConfig::default(), + )); assert_eq!(storage.relativize_path("hdfs:///a/b").unwrap(), "a/b"); } diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 0b813616bb..d7a977e7ac 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -141,11 +141,10 @@ fn build_storage_for_scheme( #[cfg(feature = "opendal-hdfs-native")] "hdfs" => { let config = crate::hdfs_native::hdfs_native_config_parse(props.clone())?; - Ok(OpenDalStorage::HdfsNative { - config: Arc::new(config), - operators: crate::HdfsNativeOperatorCache::default(), + Ok(OpenDalStorage::HdfsNative(crate::HdfsNativeStorage::new( + config, client_config, - }) + ))) } unsupported => Err(Error::new( ErrorKind::FeatureUnsupported, @@ -457,7 +456,7 @@ mod tests { let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap(); assert!( - matches!(&*resolved, OpenDalStorage::HdfsNative { .. }), + matches!(&*resolved, OpenDalStorage::HdfsNative(_)), "expected HdfsNative variant, got {resolved:?}" ); } From 6603348cc7c7cf7c50949b00f09cdb36f8e3ca35 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 13:21:38 +0300 Subject: [PATCH 21/51] perf(storage): build HDFS operators on a blocking thread The build reads the Hadoop XML config synchronously; create_operator is now async and offloads it with spawn_blocking, outside the cache lock as before. The builder itself stays sync and runtime-free. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 147 ++++++++++++++-------- crates/storage/opendal/src/lib.rs | 22 ++-- 2 files changed, 108 insertions(+), 61 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 40682aa566..bef7d309f6 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -331,9 +331,9 @@ fn poisoned(_: T) -> Error { /// Creates an operator for the path, reusing the cached one for its /// effective NameNode. -pub(crate) fn hdfs_native_create_operator<'a>( +pub(crate) async fn hdfs_native_create_operator<'a>( path: &'a str, - config: &HdfsNativeConfig, + config: &Arc, operators: &HdfsNativeOperatorCache, ) -> Result<(Operator, &'a str)> { let (name_node, relative_path) = hdfs_native_effective_name_node(config, path)?; @@ -342,11 +342,18 @@ pub(crate) fn hdfs_native_create_operator<'a>( return Ok((op, relative_path)); } - // The build reads the Hadoop XML config synchronously (~0.1 ms, once - // per NameNode), so it runs outside the lock and is not worth a - // blocking-thread hop. A racing first caller may build too; the loser - // is dropped before opening any connection. - let op = hdfs_native_operator_build(config, &name_node)?; + // The build reads the Hadoop XML config synchronously, so it runs on a + // blocking thread and outside the lock. A racing first caller may build + // too; the loser is dropped before opening any connection. + let build_config = Arc::clone(config); + let build_name_node = name_node.clone(); + let op = tokio::task::spawn_blocking(move || { + hdfs_native_operator_build(&build_config, &build_name_node) + }) + .await + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "HDFS operator build task failed").with_source(e) + })??; Ok((operators.insert(name_node, op)?, relative_path)) } @@ -663,20 +670,27 @@ mod tests { assert_eq!(hdfs_native_batch_key(&config, "hdfs:///a"), "hdfs:///a"); } - // The operator tests are plain `#[test]`s on purpose: building an - // operator must not need a runtime, as the siblings' tests also assume. - #[test] - fn test_hdfs_native_create_operator_configured_name_node_serves_logical_paths() { - let config = hdfs_native_config_parse(HashMap::from([( - HDFS_NAME_NODE.to_string(), - "hdfs://nn1:8020/,hdfs://nn2:8020/".to_string(), - )])) - .unwrap(); + // Creating an operator offloads the build to a blocking thread, so these + // run under tokio; `test_hdfs_native_operator_cache_keeps_first_insert` + // pins that the build itself needs no runtime. + #[tokio::test] + async fn test_hdfs_native_create_operator_configured_name_node_serves_logical_paths() { + let config = Arc::new( + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + "hdfs://nn1:8020/,hdfs://nn2:8020/".to_string(), + )])) + .unwrap(), + ); let operators = HdfsNativeOperatorCache::default(); // Logical and authority-less paths share the configured list's operator. - let (_, rel) = hdfs_native_create_operator("hdfs://ns-a/a/b", &config, &operators).unwrap(); - hdfs_native_create_operator("hdfs:///c", &config, &operators).unwrap(); + let (_, rel) = hdfs_native_create_operator("hdfs://ns-a/a/b", &config, &operators) + .await + .unwrap(); + hdfs_native_create_operator("hdfs:///c", &config, &operators) + .await + .unwrap(); assert_eq!(rel, "a/b"); assert_eq!(operators.len(), 1); assert!( @@ -687,35 +701,66 @@ mod tests { ); // A concrete authority is another cluster, never the configured one. - hdfs_native_create_operator("hdfs://other:9000/d", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs://other:9000/d", &config, &operators) + .await + .unwrap(); assert_eq!(operators.len(), 2); assert!(operators.get("hdfs://other:9000").unwrap().is_some()); } - #[test] - fn test_hdfs_native_create_operator_uses_path_authority() { - let config = HdfsNativeConfig::default(); + #[tokio::test] + async fn test_hdfs_native_create_operator_uses_path_authority() { + let config = Arc::new(HdfsNativeConfig::default()); let operators = HdfsNativeOperatorCache::default(); - let (_, rel) = - hdfs_native_create_operator("hdfs://nn:8020/a/b", &config, &operators).unwrap(); + let (_, rel) = hdfs_native_create_operator("hdfs://nn:8020/a/b", &config, &operators) + .await + .unwrap(); assert_eq!(rel, "a/b"); assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); } - #[test] - fn test_hdfs_native_create_operator_caches_per_name_node() { - let config = HdfsNativeConfig::default(); + #[tokio::test] + async fn test_hdfs_native_create_operator_caches_per_name_node() { + let config = Arc::new(HdfsNativeConfig::default()); let operators = HdfsNativeOperatorCache::default(); - hdfs_native_create_operator("hdfs://nn1:8020/a", &config, &operators).unwrap(); - hdfs_native_create_operator("hdfs://nn1:8020/b", &config, &operators).unwrap(); - hdfs_native_create_operator("hdfs://nn2:8020/c", &config, &operators).unwrap(); + for path in [ + "hdfs://nn1:8020/a", + "hdfs://nn1:8020/b", + "hdfs://nn2:8020/c", + ] { + hdfs_native_create_operator(path, &config, &operators) + .await + .unwrap(); + } assert_eq!(operators.len(), 2); } + #[tokio::test(flavor = "multi_thread")] + async fn test_hdfs_native_create_operator_concurrent_callers_share_one_entry() { + let config = Arc::new(HdfsNativeConfig::default()); + let operators = HdfsNativeOperatorCache::default(); + + let callers: Vec<_> = (0..8) + .map(|_| { + let (config, operators) = (config.clone(), operators.clone()); + tokio::spawn(async move { + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) + .await + .unwrap(); + }) + }) + .collect(); + for caller in callers { + caller.await.unwrap(); + } + + assert_eq!(operators.len(), 1); + } + #[test] fn test_hdfs_native_operator_cache_keeps_first_insert() { let operators = HdfsNativeOperatorCache::default(); @@ -736,16 +781,6 @@ mod tests { assert_eq!(first.info().root(), "/first/"); assert_eq!(second.info().root(), "/first/"); assert_eq!(operators.len(), 1); - - let config = HdfsNativeConfig::default(); - std::thread::scope(|scope| { - for _ in 0..8 { - scope.spawn(|| { - hdfs_native_create_operator("hdfs://other:8020/a", &config, &operators).unwrap() - }); - } - }); - assert_eq!(operators.len(), 2); } #[test] @@ -760,18 +795,26 @@ mod tests { ]; for build_runtime in flavors { let operators = HdfsNativeOperatorCache::default(); - let config = HdfsNativeConfig::default(); + let config = Arc::new(HdfsNativeConfig::default()); let runtime = build_runtime(); runtime.block_on(async { - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) + .await + .unwrap(); }); assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); - // The building runtime is gone: the entry is stale and gets rebuilt. + // The building runtime is gone: the entry is stale and the next + // runtime to use it rebuilds it. drop(runtime); assert!(operators.get("hdfs://nn:8020").unwrap().is_none()); - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); + let next = build_runtime(); + next.block_on(async { + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) + .await + .unwrap(); + }); assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); assert_eq!(operators.len(), 1); } @@ -793,8 +836,10 @@ mod tests { let baseline = metrics.num_alive_tasks(); let operators = HdfsNativeOperatorCache::default(); - let config = HdfsNativeConfig::default(); - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); + let config = Arc::new(HdfsNativeConfig::default()); + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) + .await + .unwrap(); assert_eq!(metrics.num_alive_tasks(), baseline + 1); // Dropping the cache aborts the sentinel; the abort lands once @@ -809,12 +854,14 @@ mod tests { } } - #[test] - fn test_hdfs_native_create_operator_authority_less_without_config_errors() { - let config = HdfsNativeConfig::default(); + #[tokio::test] + async fn test_hdfs_native_create_operator_authority_less_without_config_errors() { + let config = Arc::new(HdfsNativeConfig::default()); let operators = HdfsNativeOperatorCache::default(); - let err = hdfs_native_create_operator("hdfs:///a/b", &config, &operators).unwrap_err(); + let err = hdfs_native_create_operator("hdfs:///a/b", &config, &operators) + .await + .unwrap_err(); assert!(err.to_string().contains(HDFS_NAME_NODE)); } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index ddc207e136..20b94ba63a 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -394,7 +394,7 @@ impl OpenDalStorage { /// * An [`opendal::Operator`] instance used to operate on file. /// * Relative path to the root uri of [`opendal::Operator`]. #[allow(unreachable_code, unused_variables)] - pub(crate) fn create_operator<'a>( + pub(crate) async fn create_operator<'a>( &self, path: &'a impl AsRef, ) -> Result<(Operator, &'a str)> { @@ -459,7 +459,7 @@ impl OpenDalStorage { } #[cfg(feature = "opendal-hdfs-native")] OpenDalStorage::HdfsNative(storage) => { - hdfs_native_create_operator(path, &storage.config, &storage.operators)? + hdfs_native_create_operator(path, &storage.config, &storage.operators).await? } #[cfg(feature = "opendal-oss")] OpenDalStorage::Oss { config, .. } => { @@ -673,12 +673,12 @@ impl OpenDalStorage { #[async_trait] impl Storage for OpenDalStorage { async fn exists(&self, path: &str) -> Result { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; Ok(op.exists(relative_path).await.map_err(from_opendal_error)?) } async fn metadata(&self, path: &str) -> Result { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; let meta = op.stat(relative_path).await.map_err(from_opendal_error)?; Ok(FileMetadata { size: meta.content_length(), @@ -686,7 +686,7 @@ impl Storage for OpenDalStorage { } async fn read(&self, path: &str) -> Result { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; Ok(op .read(relative_path) .await @@ -695,14 +695,14 @@ impl Storage for OpenDalStorage { } async fn reader(&self, path: &str) -> Result> { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; Ok(Box::new(OpenDalReader( op.reader(relative_path).await.map_err(from_opendal_error)?, ))) } async fn write(&self, path: &str, bs: Bytes) -> Result<()> { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; op.write(relative_path, bs) .await .map_err(from_opendal_error)?; @@ -710,19 +710,19 @@ impl Storage for OpenDalStorage { } async fn writer(&self, path: &str) -> Result> { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; Ok(Box::new(OpenDalWriter::new( op.writer(relative_path).await.map_err(from_opendal_error)?, ))) } async fn delete(&self, path: &str) -> Result<()> { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; Ok(op.delete(relative_path).await.map_err(from_opendal_error)?) } async fn delete_prefix(&self, path: &str) -> Result<()> { - let (op, relative_path) = self.create_operator(&path)?; + let (op, relative_path) = self.create_operator(&path).await?; let path = if relative_path.ends_with('/') { relative_path.to_string() } else { @@ -746,7 +746,7 @@ impl Storage for OpenDalStorage { (self.relativize_path(&path)?.to_string(), entry.into_mut()) } Entry::Vacant(entry) => { - let (op, rel) = self.create_operator(&path)?; + let (op, rel) = self.create_operator(&path).await?; let rel = rel.to_string(); let deleter = op.deleter().await.map_err(from_opendal_error)?; (rel, entry.insert(deleter)) From ed23ab6b0bc3262cdf620f28086c6dfa6ab2f8c4 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 13:28:11 +0300 Subject: [PATCH 22/51] ci: run the HDFS integration tests in their own job The default suite no longer pulls the Hadoop image or waits on the HDFS health checks; a dedicated required job starts only the two HDFS containers and runs the HDFS test binary. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 48 ++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cbf9c6132..087abe08c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,9 +244,6 @@ jobs: - name: Start Docker containers if: matrix.test-suite.name == 'default' - env: - # The HDFS fixture needs host networking; opt in here (Linux only). - COMPOSE_PROFILES: hdfs run: make docker-up - name: Run tests @@ -257,9 +254,6 @@ jobs: HF_TOKEN: ${{ secrets.HF_TOKEN }} HF_BUCKET: ${{ secrets.HF_BUCKET }} HF_DATASET: ${{ secrets.HF_DATASET }} - # HDFS integration tests self-skip when unset; see - # crates/storage/opendal/tests/file_io_hdfs_test.rs - ICEBERG_TEST_HDFS_ENDPOINT: hdfs://localhost:8020 run: | if [ "${{ matrix.test-suite.name }}" = "default" ]; then cargo nextest run ${{ matrix.test-suite.args }} @@ -271,6 +265,46 @@ jobs: if: always() && matrix.test-suite.name == 'default' run: make docker-down + tests-hdfs: + needs: lint + runs-on: ubuntu-latest + name: Tests (hdfs) + env: + CARGO_INCREMENTAL: "0" + CARGO_PROFILE_DEV_DEBUG: "0" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Rust toolchain + uses: $/.github/actions/setup-builder + + - name: Cache Rust artifacts + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + key: hdfs + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Install cargo-nextest + uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 + with: + tool: cargo-nextest + + # Only the two HDFS services: the fixture needs host networking and a + # 2 GB image, so it stays out of the default suite's docker-up. + - name: Start HDFS fixture + run: docker compose -f dev/docker-compose.yaml --profile hdfs up -d --wait hdfs-namenode hdfs-datanode + + - name: Run HDFS integration tests + env: + ICEBERG_TEST_HDFS_ENDPOINT: hdfs://localhost:8020 + run: cargo nextest run -p iceberg-storage-opendal --features opendal-hdfs-native --test file_io_hdfs_test + + - name: Stop HDFS fixture + if: always() + run: docker compose -f dev/docker-compose.yaml --profile hdfs down -v --remove-orphans --timeout 0 + msrv: name: Verify MSRV needs: lint @@ -293,7 +327,7 @@ jobs: ci-required: if: ${{ always() }} - needs: [lint, clippy, build, check_standalone, build_with_no_default_features, tests, msrv] + needs: [lint, clippy, build, check_standalone, build_with_no_default_features, tests, tests-hdfs, msrv] runs-on: ubuntu-slim steps: - name: Verify CI jobs succeeded From cc9dd910e2799a1f9dc731c6fa36f2d76ef9a6e0 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 13:34:17 +0300 Subject: [PATCH 23/51] fix(storage): stop dropping HDFS properties silently PyIceberg's hdfs.user and hdfs.kerberos_ticket log a warning instead of vanishing (opendal's config has no user field), hdfs.port is validated whether or not a host is set and warns when it has no host to apply to, and a bare hadoop. key is rejected. Co-Authored-By: Claude Fable 5.1 --- crates/iceberg/src/io/storage/config/hdfs.rs | 3 +- crates/storage/opendal/Cargo.toml | 7 +- crates/storage/opendal/src/hdfs_native.rs | 106 +++++++++++++++---- 3 files changed, 91 insertions(+), 25 deletions(-) diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index a363965829..86c6db7004 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -28,7 +28,8 @@ pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; /// have no equivalent: the client reads `HADOOP_USER_NAME` and the default /// Kerberos credential cache. pub const HDFS_HOST: &str = "hdfs.host"; -/// NameNode port for [`HDFS_HOST`]; defaults to `8020`. +/// NameNode port for [`HDFS_HOST`]; defaults to `8020` and has no effect +/// without the host. pub const HDFS_PORT: &str = "hdfs.port"; /// Prefix for properties forwarded to the HDFS client configuration with the /// prefix stripped: `hadoop.dfs.client.failover.random.order` is forwarded as diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index 4f5f2e29e3..83541e89c5 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -42,7 +42,11 @@ opendal-all = [ opendal-azdls = ["opendal/services-azdls"] opendal-fs = ["opendal/services-fs"] opendal-gcs = ["opendal/services-gcs"] -opendal-hdfs-native = ["opendal/services-hdfs-native", "dep:tokio"] +opendal-hdfs-native = [ + "opendal/services-hdfs-native", + "dep:tokio", + "dep:tracing", +] opendal-hf = ["opendal/services-hf"] opendal-memory = ["opendal/services-memory"] opendal-oss = ["opendal/services-oss"] @@ -61,6 +65,7 @@ reqsign-aws-v4 = { version = "3.0.0", optional = true } reqsign-core = { version = "3.0.0", optional = true } serde = { workspace = true } tokio = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } typetag = { workspace = true } url = { workspace = true } diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index bef7d309f6..3a0ede7bc1 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -36,6 +36,8 @@ use crate::utils::from_opendal_error; /// Hadoop's default filesystem, which serves authority-less paths. const FS_DEFAULT_FS: &str = "fs.defaultFS"; const HDFS_DEFAULT_PORT: u16 = 8020; +/// PyIceberg keys with no equivalent in opendal's config. +const HDFS_UNSUPPORTED_KEYS: [&str; 2] = ["hdfs.user", "hdfs.kerberos_ticket"]; /// `hdfs-native` dials a NameNode as a socket address and has no default /// port, so anything without one can only be a logical nameservice name. @@ -73,8 +75,39 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result } } - let host = m.remove(HDFS_HOST).map(|s| s.trim().to_string()); - let port = m.remove(HDFS_PORT).map(|s| s.trim().to_string()); + // A config carried over from PyIceberg would otherwise change identity + // silently; the client reads `HADOOP_USER_NAME` and the Kerberos cache. + for key in HDFS_UNSUPPORTED_KEYS { + if m.remove(key).is_some() { + tracing::warn!("`{key}` is not supported by the hdfs-native backend and is ignored"); + } + } + if m.contains_key(HDFS_HADOOP_CONF_PREFIX) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{HDFS_HADOOP_CONF_PREFIX}`: a Hadoop key must follow the prefix" + ), + )); + } + + let host = m + .remove(HDFS_HOST) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + let port = m + .remove(HDFS_PORT) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .map(|port| { + port.parse::().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid `{HDFS_PORT}`: {port}: {e}"), + ) + }) + }) + .transpose()?; let mut options: HashMap = m .into_iter() @@ -86,25 +119,23 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result // PyIceberg's `hdfs.host`/`hdfs.port` name the filesystem for // authority-less paths, which is what Hadoop's `fs.defaultFS` means; an // explicit `hadoop.fs.defaultFS` wins. - if let Some(host) = host.filter(|s| !s.is_empty()) { - let port = match port.filter(|s| !s.is_empty()) { - Some(port) => port.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid `{HDFS_PORT}`: {port}: {e}"), - ) - })?, - None => HDFS_DEFAULT_PORT, - }; - // An IPv6 literal needs brackets in a URI authority. - let host = if host.contains(':') && !host.starts_with('[') { - format!("[{host}]") - } else { - host - }; - options - .entry(FS_DEFAULT_FS.to_string()) - .or_insert_with(|| format!("hdfs://{host}:{port}")); + match host { + Some(host) => { + // An IPv6 literal needs brackets in a URI authority. + let host = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]") + } else { + host + }; + let port = port.unwrap_or(HDFS_DEFAULT_PORT); + options + .entry(FS_DEFAULT_FS.to_string()) + .or_insert_with(|| format!("hdfs://{host}:{port}")); + } + None if port.is_some() => { + tracing::warn!("`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored"); + } + None => {} } if !options.is_empty() { cfg.options = Some(options); @@ -482,8 +513,37 @@ mod tests { None ); assert_eq!(parse(&[(HDFS_PORT, "9000")]).unwrap(), None); - let err = parse(&[(HDFS_HOST, "nn"), (HDFS_PORT, "x")]).unwrap_err(); - assert!(err.to_string().contains(HDFS_PORT)); + // A bad port is rejected whether or not a host accompanies it. + for props in [ + &[(HDFS_HOST, "nn"), (HDFS_PORT, "x")][..], + &[(HDFS_PORT, "x")][..], + &[(HDFS_PORT, "70000")][..], + ] { + let err = parse(props).unwrap_err(); + assert!(err.to_string().contains(HDFS_PORT), "{props:?}: {err}"); + } + } + + #[test] + fn test_hdfs_native_config_parse_rejects_bare_prefix_and_drops_unsupported_keys() { + let err = hdfs_native_config_parse(HashMap::from([( + HDFS_HADOOP_CONF_PREFIX.to_string(), + "x".to_string(), + )])) + .unwrap_err(); + assert!(err.to_string().contains(HDFS_HADOOP_CONF_PREFIX), "{err}"); + + // PyIceberg's identity keys are ignored (with a warning), never + // forwarded as Hadoop options. + let cfg = hdfs_native_config_parse(HashMap::from([ + ("hdfs.user".to_string(), "alice".to_string()), + ( + "hdfs.kerberos_ticket".to_string(), + "/tmp/krb5cc".to_string(), + ), + ])) + .unwrap(); + assert_eq!(cfg.options, None); } #[test] From f83db4ebc34f84a2a329b37cb388140a24197d19 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 13:50:37 +0300 Subject: [PATCH 24/51] test(storage): derive the second NameNode spelling from the endpoint host Resolving the host to its IPv4 address gives any hostname endpoint a second batch key; an IP-literal endpoint skips the test explicitly instead of passing as a no-op. Co-Authored-By: Claude Fable 5.1 --- .../opendal/tests/file_io_hdfs_test.rs | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index 907151cafb..1c0c9e7360 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -30,6 +30,7 @@ #[cfg(feature = "opendal-hdfs-native")] mod tests { + use std::net::ToSocketAddrs; use std::sync::Arc; use bytes::Bytes; @@ -67,6 +68,22 @@ mod tests { ) } + /// The endpoint with its host replaced by that host's IPv4 address: a + /// second spelling of the same NameNode, so a second batch key. `None` + /// when the endpoint already uses an IP literal. + fn alternate_endpoint(endpoint: &str) -> Option { + let url = url::Url::parse(endpoint).ok()?; + let host = url.host_str()?; + if host.parse::().is_ok() { + return None; + } + let addr = (host, url.port()?) + .to_socket_addrs() + .ok()? + .find(std::net::SocketAddr::is_ipv4)?; + Some(format!("hdfs://{addr}")) + } + #[tokio::test] async fn test_file_io_hdfs_exists() { require_hdfs!(); @@ -183,9 +200,15 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete_stream_two_name_nodes() { require_hdfs!(); + let endpoint = get_hdfs_endpoint(); + let Some(alternate) = alternate_endpoint(&endpoint) else { + eprintln!("Skipping HDFS test: {endpoint} has no second spelling"); + return; + }; let file_io = get_file_io(); let dir = test_path("test_file_io_hdfs_delete_stream_two_name_nodes"); - let alt_dir = dir.replacen("localhost", "127.0.0.1", 1); + let alt_dir = dir.replacen(&endpoint, &alternate, 1); + assert_ne!(dir, alt_dir); let paths = vec![format!("{dir}/a"), format!("{alt_dir}/b")]; for path in &paths { From 9c62f4c7048666aa506f22321a146a18481aff69 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 16:13:09 +0300 Subject: [PATCH 25/51] chore: record tracing as a dependency of iceberg-storage-opendal in Cargo.lock Co-Authored-By: Claude Fable 5.1 --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 4e85b1f063..ab75bda3b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3090,6 +3090,7 @@ dependencies = [ "serde_json", "tempfile", "tokio", + "tracing", "typetag", "url", ] From a0d21942377fe0b23f0a5111461670b7b1648e2b Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 17:47:43 +0300 Subject: [PATCH 26/51] fix(storage): report a missing tokio runtime instead of panicking in HDFS create_operator The runtime handle is resolved up front and passed into the cache, so the sentinel is no longer optional and the unreachable no-runtime branch is gone. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 85 +++++++++++++++-------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 3a0ede7bc1..7ed1213957 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -264,8 +264,8 @@ impl HdfsNativeStorage { /// Operators cached per effective NameNode: each holds an `hdfs-native` /// client with live RPC connections, whose tasks run on the tokio runtime -/// current when it was built (a private one when built outside any). An -/// entry is rebuilt once that runtime is gone, as `hdfs-native` panics when +/// current when it was built. An entry is rebuilt once that runtime is +/// gone, as `hdfs-native` panics when /// it spawns onto a dead one. The cache lives as long as the storage that /// owns it (clones share it). #[derive(Clone, Debug, Default)] @@ -274,8 +274,7 @@ pub(crate) struct HdfsNativeOperatorCache(Arc, + sentinel: RuntimeSentinel, } /// A task parked on the building runtime that owns the token, so the token @@ -306,17 +305,15 @@ impl Drop for RuntimeSentinel { } impl CachedOperator { - fn new(operator: Operator) -> Self { - let sentinel = Handle::try_current() - .ok() - .map(|handle| RuntimeSentinel::spawn(&handle)); - Self { operator, sentinel } + fn new(operator: Operator, handle: &Handle) -> Self { + Self { + operator, + sentinel: RuntimeSentinel::spawn(handle), + } } fn runtime_alive(&self) -> bool { - self.sentinel - .as_ref() - .is_none_or(|sentinel| sentinel.alive.strong_count() > 0) + self.sentinel.alive.strong_count() > 0 } } @@ -333,18 +330,18 @@ impl HdfsNativeOperatorCache { /// Inserts `op` unless a concurrent caller got there first, returning /// whichever operator the cache now holds; a stale entry is replaced. - fn insert(&self, name_node: String, op: Operator) -> Result { + fn insert(&self, name_node: String, op: Operator, handle: &Handle) -> Result { let mut operators = self.0.write().map_err(poisoned)?; match operators.entry(name_node) { Entry::Occupied(entry) if entry.get().runtime_alive() => { Ok(entry.get().operator.clone()) } Entry::Occupied(mut entry) => { - entry.insert(CachedOperator::new(op.clone())); + entry.insert(CachedOperator::new(op.clone(), handle)); Ok(op) } Entry::Vacant(entry) => { - entry.insert(CachedOperator::new(op.clone())); + entry.insert(CachedOperator::new(op.clone(), handle)); Ok(op) } } @@ -373,19 +370,27 @@ pub(crate) async fn hdfs_native_create_operator<'a>( return Ok((op, relative_path)); } + // Every operator in this crate needs a tokio runtime for its I/O (the + // timeout layer), so say so instead of panicking in `spawn_blocking`. + let handle = Handle::try_current().map_err(|_| { + Error::new( + ErrorKind::FeatureUnsupported, + "HDFS storage requires a tokio runtime", + ) + })?; + // The build reads the Hadoop XML config synchronously, so it runs on a // blocking thread and outside the lock. A racing first caller may build // too; the loser is dropped before opening any connection. let build_config = Arc::clone(config); let build_name_node = name_node.clone(); - let op = tokio::task::spawn_blocking(move || { - hdfs_native_operator_build(&build_config, &build_name_node) - }) - .await - .map_err(|e| { - Error::new(ErrorKind::Unexpected, "HDFS operator build task failed").with_source(e) - })??; - Ok((operators.insert(name_node, op)?, relative_path)) + let op = handle + .spawn_blocking(move || hdfs_native_operator_build(&build_config, &build_name_node)) + .await + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "HDFS operator build task failed").with_source(e) + })??; + Ok((operators.insert(name_node, op, &handle)?, relative_path)) } /// Returns the `delete_stream` grouping key for a path: the effective @@ -731,8 +736,8 @@ mod tests { } // Creating an operator offloads the build to a blocking thread, so these - // run under tokio; `test_hdfs_native_operator_cache_keeps_first_insert` - // pins that the build itself needs no runtime. + // run under tokio; `test_hdfs_native_operator_build_needs_no_runtime` + // pins that the build itself does not. #[tokio::test] async fn test_hdfs_native_create_operator_configured_name_node_serves_logical_paths() { let config = Arc::new( @@ -821,9 +826,33 @@ mod tests { assert_eq!(operators.len(), 1); } + /// The build itself needs no runtime; only creating a cached operator does. + #[test] + fn test_hdfs_native_operator_build_needs_no_runtime() { + let config = HdfsNativeConfig::default(); + hdfs_native_operator_build(&config, "hdfs://nn:8020").unwrap(); + } + #[test] - fn test_hdfs_native_operator_cache_keeps_first_insert() { + fn test_hdfs_native_create_operator_without_runtime_errors() { + let config = Arc::new(HdfsNativeConfig::default()); + let operators = HdfsNativeOperatorCache::default(); + + let err = futures::executor::block_on(hdfs_native_create_operator( + "hdfs://nn:8020/a", + &config, + &operators, + )) + .unwrap_err(); + + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + assert_eq!(operators.len(), 0); + } + + #[tokio::test] + async fn test_hdfs_native_operator_cache_keeps_first_insert() { let operators = HdfsNativeOperatorCache::default(); + let handle = Handle::current(); let build = |root: &str| { let mut config = hdfs_native_config_parse(HashMap::new()).unwrap(); config.root = Some(root.to_string()); @@ -833,10 +862,10 @@ mod tests { // Two callers racing for one NameNode: the first insert wins and both // get the cached operator. let first = operators - .insert("hdfs://nn:8020".to_string(), build("/first")) + .insert("hdfs://nn:8020".to_string(), build("/first"), &handle) .unwrap(); let second = operators - .insert("hdfs://nn:8020".to_string(), build("/second")) + .insert("hdfs://nn:8020".to_string(), build("/second"), &handle) .unwrap(); assert_eq!(first.info().root(), "/first/"); assert_eq!(second.info().root(), "/first/"); From ca922405e277226bff4604d97373593e3e660edf Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 17:55:03 +0300 Subject: [PATCH 27/51] fix(storage): normalize every HDFS NameNode spelling to hdfs://host:port One parser classifies configured entries, path authorities and fs.defaultFS: other schemes, logical names, port 0, userinfo, paths and unbracketed IPv6 are rejected, and bare and prefixed spellings share one cache key. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 141 +++++++++++++++------- 1 file changed, 95 insertions(+), 46 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 7ed1213957..11d31d4e7b 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -39,12 +39,26 @@ const HDFS_DEFAULT_PORT: u16 = 8020; /// PyIceberg keys with no equivalent in opendal's config. const HDFS_UNSUPPORTED_KEYS: [&str; 2] = ["hdfs.user", "hdfs.kerberos_ticket"]; -/// `hdfs-native` dials a NameNode as a socket address and has no default -/// port, so anything without one can only be a logical nameservice name. -fn hdfs_native_has_port(name_node: &str) -> bool { - name_node - .rsplit_once(':') - .is_some_and(|(_, port)| port.parse::().is_ok()) +/// Normalizes one NameNode spelling to `hdfs://host:port`, the form path +/// authorities take, so every source shares cache keys. `hdfs-native` dials +/// a socket address and has no default port, so anything else is `None`: +/// another scheme, a logical name, port 0, userinfo, a path. +fn hdfs_native_name_node(entry: &str) -> Option { + let rest = entry.trim().trim_end_matches('/'); + let rest = rest.strip_prefix("hdfs://").unwrap_or(rest); + if rest.is_empty() || rest.contains("://") { + return None; + } + let url = Url::parse(&format!("hdfs://{rest}")).ok()?; + let (host, port) = (url.host_str()?, url.port()?); + let plain = host.is_empty() + || port == 0 + || !url.username().is_empty() + || url.password().is_some() + || !matches!(url.path(), "" | "/") + || url.query().is_some() + || url.fragment().is_some(); + (!plain).then(|| format!("hdfs://{host}:{port}")) } /// Parse iceberg properties to [`HdfsNativeConfig`]. @@ -57,19 +71,21 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result // builder's empty-string guard and `Some("")` would shadow the // path-authority fallback below. if let Some(name_node) = m.remove(HDFS_NAME_NODE) { - let entries: Vec<&str> = name_node + let entries = name_node .split(',') - .map(|entry| entry.trim().trim_end_matches('/')) + .map(str::trim) .filter(|entry| !entry.is_empty()) - .collect(); - // Each entry is dialed as `host:port` (`hdfs://` optional); a portless - // one would fail only at the first I/O. - if let Some(entry) = entries.iter().find(|entry| !hdfs_native_has_port(entry)) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid `{HDFS_NAME_NODE}` entry: {entry}, expected host:port"), - )); - } + .map(|entry| { + hdfs_native_name_node(entry).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{HDFS_NAME_NODE}` entry: {entry}, expected host:port (hdfs:// optional)" + ), + ) + }) + }) + .collect::>>()?; if !entries.is_empty() { cfg.name_node = Some(entries.join(",")); } @@ -198,25 +214,21 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( ) }; let name_node = match authority { - Some(authority) if hdfs_native_has_port(&authority) => authority, + Some(authority) if hdfs_native_name_node(&authority).is_some() => authority, Some(logical) => config.name_node.clone().ok_or_else(|| { let logical = logical.trim_start_matches("hdfs://"); invalid(format!( "logical nameservice `{logical}` requires `{HDFS_NAME_NODE}`" )) })?, - None => match config - .name_node - .clone() - .or_else(|| hdfs_native_default_fs(config)) - { - Some(name_node) if hdfs_native_has_port(&name_node) => name_node, - Some(logical) => { - return Err(invalid(format!( - "`{FS_DEFAULT_FS}` {logical} has no port, a logical nameservice requires `{HDFS_NAME_NODE}`" - ))); - } - None => { + None => match (&config.name_node, hdfs_native_default_fs(config)) { + (Some(name_node), _) => name_node.clone(), + (None, Some(default_fs)) => hdfs_native_name_node(default_fs).ok_or_else(|| { + invalid(format!( + "`{FS_DEFAULT_FS}` {default_fs} is not an HDFS host:port, a logical nameservice requires `{HDFS_NAME_NODE}`" + )) + })?, + (None, None) => { return Err(invalid(format!( "authority-less paths require `{HDFS_NAME_NODE}` or `{HDFS_HOST}`" ))); @@ -226,18 +238,14 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( Ok((name_node, relative_path)) } -/// `fs.defaultFS` from the forwarded options, when it is an HDFS URI. -fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option { +/// `fs.defaultFS` from the forwarded options, as written. +fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option<&str> { config .options .as_ref()? .get(FS_DEFAULT_FS) - .map(|s| s.trim().trim_end_matches('/')) - .filter(|s| { - s.strip_prefix("hdfs://") - .is_some_and(|rest| !rest.is_empty()) - }) - .map(str::to_string) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) } /// State of [`OpenDalStorage::HdfsNative`](crate::OpenDalStorage::HdfsNative): @@ -560,20 +568,32 @@ mod tests { )])) }; - // `hdfs://` is optional: Hadoop's own rpc-address format is bare. - for value in [ - "nn:8020", - "hdfs://nn:8020", - "[::1]:8020", - "hdfs://nn1:8020,nn2:8020", + // `hdfs://` is optional (Hadoop's own rpc-address format is bare) and + // every entry is normalized to it, so it keys the cache like a path + // authority does. + for (value, normalized) in [ + ("nn:8020", "hdfs://nn:8020"), + ("hdfs://nn:8020/", "hdfs://nn:8020"), + ("[::1]:8020", "hdfs://[::1]:8020"), + ( + "hdfs://nn1:8020, nn2:8020", + "hdfs://nn1:8020,hdfs://nn2:8020", + ), ] { - assert!(parse(value).is_ok(), "{value}"); + assert_eq!(parse(value).unwrap().name_node.as_deref(), Some(normalized)); } for value in [ "hdfs://ns1", "nn", "nn:x", + "nn:0", + "nn:+80", + "::1", "hdfs://[::1]", + "viewfs://nn:8020", + "foo://nn:8020", + "user@nn:8020", + "nn:8020/path", "hdfs://nn1:8020,nn2", ] { let err = parse(value).unwrap_err().to_string(); @@ -599,7 +619,36 @@ mod tests { } // Portless: a logical nameservice, which only `hdfs.name-node` resolves. let err = hdfs_native_effective_name_node(&parse("hdfs://ns1"), "hdfs:///a").unwrap_err(); - assert!(err.to_string().contains("has no port"), "{err}"); + assert!( + err.to_string().contains(FS_DEFAULT_FS) && err.to_string().contains(HDFS_NAME_NODE), + "{err}" + ); + // Normalized like every other source. + let (nn, _) = hdfs_native_effective_name_node(&parse("nn:9000"), "hdfs:///a").unwrap(); + assert_eq!(nn, "hdfs://nn:9000"); + } + + /// A bare configured NameNode and the same NameNode as a path authority + /// share one cache entry. + #[tokio::test] + async fn test_hdfs_native_create_operator_dedupes_spellings() { + let config = Arc::new( + hdfs_native_config_parse(HashMap::from([( + HDFS_NAME_NODE.to_string(), + "nn:8020".to_string(), + )])) + .unwrap(), + ); + let operators = HdfsNativeOperatorCache::default(); + + for path in ["hdfs:///a", "hdfs://nn:8020/b"] { + hdfs_native_create_operator(path, &config, &operators) + .await + .unwrap(); + } + + assert_eq!(operators.len(), 1); + assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); } #[test] From ce78ee33cbb64ce83dc7a9c4a30a4720e630ff92 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 18:18:26 +0300 Subject: [PATCH 28/51] feat(storage): declare HDFS nameservices with hdfs.name-node. The key expands to Hadoop's dfs.ha.namenodes and dfs.namenode.rpc-address keys in the forwarded options and the resolver reads them back, so nameservices declared through hadoop.* work the same way. Plain hdfs.name-node stays the documented single-cluster default; an undeclared name without it is a pointed error. Co-Authored-By: Claude Fable 5.1 --- crates/iceberg/src/io/storage/config/hdfs.rs | 14 +- crates/storage/opendal/src/hdfs_native.rs | 280 +++++++++++++++++-- 2 files changed, 270 insertions(+), 24 deletions(-) diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index 86c6db7004..e61f9661bd 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -18,9 +18,17 @@ //! HDFS storage configuration. /// HDFS NameNode RPC endpoint(s) as `host:port`, e.g. `hdfs://namenode:8020`; -/// a comma-separated list enables HA failover. Resolves logical nameservice -/// authorities (no port) and authority-less paths; as in Hadoop, an authority -/// with a port is always used as is. +/// a comma-separated list enables HA failover. Resolves authority-less paths +/// and, as the single-cluster default, every logical nameservice authority +/// (no port) that is not declared on its own; as in Hadoop, an authority with +/// a port is always used as is. +/// +/// A catalog spanning several HDFS clusters must declare each nameservice: +/// `hdfs.name-node.` takes the same list and is sugar for +/// Hadoop's `dfs.ha.namenodes.` and +/// `dfs.namenode.rpc-address..`, which are honored as well +/// when passed through [`HDFS_HADOOP_CONF_PREFIX`]. With only the plain key +/// set, an unknown or mistyped nameservice routes to that one cluster. pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; /// NameNode host for authority-less paths, as in PyIceberg; paths that carry /// an authority ignore it, as they do there. Combined with [`HDFS_PORT`] into diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 11d31d4e7b..3c9395bc83 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -38,6 +38,10 @@ const FS_DEFAULT_FS: &str = "fs.defaultFS"; const HDFS_DEFAULT_PORT: u16 = 8020; /// PyIceberg keys with no equivalent in opendal's config. const HDFS_UNSUPPORTED_KEYS: [&str; 2] = ["hdfs.user", "hdfs.kerberos_ticket"]; +/// Hadoop's keys declaring an HA nameservice: `hdfs.name-node.` +/// expands to them, and they are honored as well when passed through `hadoop.`. +const HA_NAMENODES_PREFIX: &str = "dfs.ha.namenodes"; +const HA_NAMENODE_RPC_ADDRESS_PREFIX: &str = "dfs.namenode.rpc-address"; /// Normalizes one NameNode spelling to `hdfs://host:port`, the form path /// authorities take, so every source shares cache keys. `hdfs-native` dials @@ -61,6 +65,26 @@ fn hdfs_native_name_node(entry: &str) -> Option { (!plain).then(|| format!("hdfs://{host}:{port}")) } +/// Parses a comma-separated NameNode list, each entry normalized; an empty +/// list is `Ok` and empty. +fn hdfs_native_name_node_list(property: &str, value: &str) -> Result> { + value + .split(',') + .map(str::trim) + .filter(|entry| !entry.is_empty()) + .map(|entry| { + hdfs_native_name_node(entry).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{property}` entry: {entry}, expected host:port (hdfs:// optional)" + ), + ) + }) + }) + .collect() +} + /// Parse iceberg properties to [`HdfsNativeConfig`]. pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result { let mut cfg = HdfsNativeConfig::default(); @@ -71,26 +95,42 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result // builder's empty-string guard and `Some("")` would shadow the // path-authority fallback below. if let Some(name_node) = m.remove(HDFS_NAME_NODE) { - let entries = name_node - .split(',') - .map(str::trim) - .filter(|entry| !entry.is_empty()) - .map(|entry| { - hdfs_native_name_node(entry).ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Invalid `{HDFS_NAME_NODE}` entry: {entry}, expected host:port (hdfs:// optional)" - ), - ) - }) - }) - .collect::>>()?; + let entries = hdfs_native_name_node_list(HDFS_NAME_NODE, &name_node)?; if !entries.is_empty() { cfg.name_node = Some(entries.join(",")); } } + // `hdfs.name-node.` is sugar for Hadoop's own declaration of + // an HA nameservice, which the resolver reads back from the options. + let nameservice_prefix = format!("{HDFS_NAME_NODE}."); + let declared_keys: Vec = m + .keys() + .filter(|key| key.starts_with(&nameservice_prefix)) + .cloned() + .collect(); + let mut declared = Vec::new(); + for key in declared_keys { + let value = m.remove(&key).unwrap_or_default(); + let nameservice = key[nameservice_prefix.len()..].to_string(); + if nameservice.is_empty() || nameservice.chars().any(char::is_whitespace) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid property `{key}`: a nameservice name must follow `{nameservice_prefix}`" + ), + )); + } + let entries = hdfs_native_name_node_list(&key, &value)?; + if entries.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid property `{key}`: no NameNodes"), + )); + } + declared.push((nameservice, entries)); + } + // A config carried over from PyIceberg would otherwise change identity // silently; the client reads `HADOOP_USER_NAME` and the Kerberos cache. for key in HDFS_UNSUPPORTED_KEYS { @@ -132,6 +172,20 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result .map(|stripped| (stripped.to_string(), value)) }) .collect(); + // Explicit `hadoop.` keys win over the sugar. + for (nameservice, entries) in declared { + let ids: Vec = (0..entries.len()).map(|i| format!("nn{i}")).collect(); + options + .entry(format!("{HA_NAMENODES_PREFIX}.{nameservice}")) + .or_insert_with(|| ids.join(",")); + for (id, entry) in ids.iter().zip(&entries) { + options + .entry(format!( + "{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}" + )) + .or_insert_with(|| entry.trim_start_matches("hdfs://").to_string()); + } + } // PyIceberg's `hdfs.host`/`hdfs.port` name the filesystem for // authority-less paths, which is what Hadoop's `fs.defaultFS` means; an // explicit `hadoop.fs.defaultFS` wins. @@ -215,12 +269,17 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( }; let name_node = match authority { Some(authority) if hdfs_native_name_node(&authority).is_some() => authority, - Some(logical) => config.name_node.clone().ok_or_else(|| { - let logical = logical.trim_start_matches("hdfs://"); - invalid(format!( - "logical nameservice `{logical}` requires `{HDFS_NAME_NODE}`" - )) - })?, + Some(logical) => { + let nameservice = logical.trim_start_matches("hdfs://"); + match hdfs_native_nameservice(config, nameservice)? { + Some(name_node) => name_node, + None => config.name_node.clone().ok_or_else(|| { + invalid(format!( + "logical nameservice `{nameservice}` is not declared; set `{HDFS_NAME_NODE}.{nameservice}`, or `{HDFS_NAME_NODE}` for a single cluster" + )) + })?, + } + } None => match (&config.name_node, hdfs_native_default_fs(config)) { (Some(name_node), _) => name_node.clone(), (None, Some(default_fs)) => hdfs_native_name_node(default_fs).ok_or_else(|| { @@ -248,6 +307,44 @@ fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option<&str> { .filter(|s| !s.is_empty()) } +/// The NameNodes that Hadoop's keys in the forwarded options declare for a +/// nameservice, if any. A declaration with a missing or malformed address is +/// an error rather than a silent fallback. +fn hdfs_native_nameservice(config: &HdfsNativeConfig, nameservice: &str) -> Result> { + let Some(options) = config.options.as_ref() else { + return Ok(None); + }; + let Some(ids) = options.get(&format!("{HA_NAMENODES_PREFIX}.{nameservice}")) else { + return Ok(None); + }; + let name_nodes = ids + .split(',') + .map(str::trim) + .filter(|id| !id.is_empty()) + .map(|id| { + let key = format!("{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}"); + options + .get(&key) + .and_then(|value| hdfs_native_name_node(value)) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Nameservice `{nameservice}` declares NameNode `{id}` but `{key}` is missing or not host:port" + ), + ) + }) + }) + .collect::>>()?; + if name_nodes.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Nameservice `{nameservice}` declares no NameNodes"), + )); + } + Ok(Some(name_nodes.join(","))) +} + /// State of [`OpenDalStorage::HdfsNative`](crate::OpenDalStorage::HdfsNative): /// the parsed configuration and the per-NameNode operator cache. Only the /// storage factories build it. @@ -651,6 +748,147 @@ mod tests { assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); } + #[test] + fn test_hdfs_native_config_parse_declares_nameservices() { + let cfg = hdfs_native_config_parse(HashMap::from([ + ( + format!("{HDFS_NAME_NODE}.ns-a"), + "nn1:8020, hdfs://nn2:8020/".to_string(), + ), + ("hadoop.dfs.ha.namenodes.ns-b".to_string(), "x".to_string()), + ( + "hadoop.dfs.namenode.rpc-address.ns-b.x".to_string(), + "nn3:8020".to_string(), + ), + ])) + .unwrap(); + let options = cfg.options.clone().unwrap(); + + // The sugar expands to Hadoop's keys... + assert_eq!( + options.get("dfs.ha.namenodes.ns-a").map(String::as_str), + Some("nn0,nn1") + ); + assert_eq!( + options + .get("dfs.namenode.rpc-address.ns-a.nn0") + .map(String::as_str), + Some("nn1:8020") + ); + assert_eq!( + options + .get("dfs.namenode.rpc-address.ns-a.nn1") + .map(String::as_str), + Some("nn2:8020") + ); + // ...and the resolver reads both forms back. + assert_eq!( + hdfs_native_nameservice(&cfg, "ns-a").unwrap().as_deref(), + Some("hdfs://nn1:8020,hdfs://nn2:8020") + ); + assert_eq!( + hdfs_native_nameservice(&cfg, "ns-b").unwrap().as_deref(), + Some("hdfs://nn3:8020") + ); + assert_eq!(hdfs_native_nameservice(&cfg, "ns-c").unwrap(), None); + + for (key, value) in [ + (format!("{HDFS_NAME_NODE}."), "nn:8020"), + (format!("{HDFS_NAME_NODE}.ns"), "nn"), + (format!("{HDFS_NAME_NODE}.ns"), " "), + ] { + let err = hdfs_native_config_parse(HashMap::from([(key.clone(), value.to_string())])) + .unwrap_err(); + assert!(err.to_string().contains(&key), "{key}={value}: {err}"); + } + // A declaration with a broken address is an error, not a silent fallback. + let broken = hdfs_native_config_parse(HashMap::from([( + "hadoop.dfs.ha.namenodes.ns-d".to_string(), + "a".to_string(), + )])) + .unwrap(); + assert!(hdfs_native_nameservice(&broken, "ns-d").is_err()); + } + + #[test] + fn test_hdfs_native_effective_name_node_declared_nameservices() { + let parse = |props: &[(String, &str)]| { + hdfs_native_config_parse( + props + .iter() + .map(|(k, v)| (k.clone(), v.to_string())) + .collect(), + ) + .unwrap() + }; + let ns_a = ( + format!("{HDFS_NAME_NODE}.ns-a"), + "hdfs://a1:8020,hdfs://a2:8020", + ); + let declared_only = parse(std::slice::from_ref(&ns_a)); + let with_default = parse(&[ns_a.clone(), (HDFS_NAME_NODE.to_string(), "hdfs://d:8020")]); + + // A declared nameservice resolves to its own list, ahead of the plain key. + for config in [&declared_only, &with_default] { + let (nn, rel) = hdfs_native_effective_name_node(config, "hdfs://ns-a/x").unwrap(); + assert_eq!((nn.as_str(), rel), ("hdfs://a1:8020,hdfs://a2:8020", "x")); + } + // An undeclared one falls back to the plain key only when it is set. + let (nn, _) = hdfs_native_effective_name_node(&with_default, "hdfs://ns-b/x").unwrap(); + assert_eq!(nn, "hdfs://d:8020"); + let err = hdfs_native_effective_name_node(&declared_only, "hdfs://ns-b/x").unwrap_err(); + assert!( + err.to_string().contains("ns-b") && err.to_string().contains(HDFS_NAME_NODE), + "{err}" + ); + // A concrete authority is never redirected. + let (nn, _) = + hdfs_native_effective_name_node(&with_default, "hdfs://other:9000/x").unwrap(); + assert_eq!(nn, "hdfs://other:9000"); + } + + #[tokio::test] + async fn test_hdfs_native_create_operator_per_nameservice() { + let config = Arc::new( + hdfs_native_config_parse(HashMap::from([ + ( + format!("{HDFS_NAME_NODE}.ns-a"), + "hdfs://a:8020".to_string(), + ), + ( + format!("{HDFS_NAME_NODE}.ns-b"), + "hdfs://b:8020".to_string(), + ), + (HDFS_NAME_NODE.to_string(), "hdfs://d:8020".to_string()), + ])) + .unwrap(), + ); + let operators = HdfsNativeOperatorCache::default(); + + for path in [ + "hdfs://ns-a/x", + "hdfs://ns-b/y", + "hdfs://ns-c/z", + "hdfs:///w", + ] { + hdfs_native_create_operator(path, &config, &operators) + .await + .unwrap(); + } + + // ns-a, ns-b, and the default serving both the undeclared ns-c and + // the authority-less path. + assert_eq!(operators.len(), 3); + assert_eq!( + hdfs_native_batch_key(&config, "hdfs://ns-c/z"), + "hdfs://d:8020" + ); + assert_eq!( + hdfs_native_batch_key(&config, "hdfs://ns-b/y"), + "hdfs://b:8020" + ); + } + #[test] fn test_hdfs_native_effective_name_node_precedence() { let configured = hdfs_native_config_parse(HashMap::from([ From 4a9cc25353748ce1a7e410c24333016075d07556 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 22:42:19 +0300 Subject: [PATCH 29/51] fix(storage): require a declaration for every logical HDFS nameservice Plain hdfs.name-node serves authority-less paths only; an undeclared logical authority is an error naming the key to set instead of a guess that could reach the wrong cluster. Co-Authored-By: Claude Fable 5.1 --- crates/iceberg/src/io/storage/config/hdfs.rs | 18 ++--- crates/storage/opendal/src/hdfs_native.rs | 78 +++++++++---------- crates/storage/opendal/src/lib.rs | 7 +- .../opendal/tests/file_io_hdfs_test.rs | 11 ++- 4 files changed, 56 insertions(+), 58 deletions(-) diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index e61f9661bd..e2f1d3cfe8 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -18,17 +18,15 @@ //! HDFS storage configuration. /// HDFS NameNode RPC endpoint(s) as `host:port`, e.g. `hdfs://namenode:8020`; -/// a comma-separated list enables HA failover. Resolves authority-less paths -/// and, as the single-cluster default, every logical nameservice authority -/// (no port) that is not declared on its own; as in Hadoop, an authority with -/// a port is always used as is. -/// -/// A catalog spanning several HDFS clusters must declare each nameservice: -/// `hdfs.name-node.` takes the same list and is sugar for -/// Hadoop's `dfs.ha.namenodes.` and +/// a comma-separated list enables HA failover. Serves authority-less paths +/// (`hdfs:///path`) only. As in Hadoop, an authority with a port is always +/// used as is, and a logical nameservice authority (`hdfs://ns1/path`, no +/// port) must be declared: `hdfs.name-node.` takes the same +/// list and is sugar for Hadoop's `dfs.ha.namenodes.` and /// `dfs.namenode.rpc-address..`, which are honored as well -/// when passed through [`HDFS_HADOOP_CONF_PREFIX`]. With only the plain key -/// set, an unknown or mistyped nameservice routes to that one cluster. +/// when passed through [`HDFS_HADOOP_CONF_PREFIX`]. An undeclared nameservice +/// is an error rather than a guess, and nameservices defined only in +/// `$HADOOP_CONF_DIR` are not visible here, so declare them in properties. pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; /// NameNode host for authority-less paths, as in PyIceberg; paths that carry /// an authority ignore it, as they do there. Combined with [`HDFS_PORT`] into diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 3c9395bc83..9c6005b37a 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -252,7 +252,7 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str /// Resolves the effective NameNode for a path, plus the relative path. As in /// Hadoop, an authority with a port is used as is; a logical nameservice -/// authority (no port) resolves through `hdfs.name-node`, and an +/// authority (no port) resolves through its declaration, and an /// authority-less path through `hdfs.name-node`, else `fs.defaultFS`. The /// operator cache, `delete_stream` batching and `relativize_path` all go /// through this, so they cannot drift apart. @@ -271,14 +271,11 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( Some(authority) if hdfs_native_name_node(&authority).is_some() => authority, Some(logical) => { let nameservice = logical.trim_start_matches("hdfs://"); - match hdfs_native_nameservice(config, nameservice)? { - Some(name_node) => name_node, - None => config.name_node.clone().ok_or_else(|| { - invalid(format!( - "logical nameservice `{nameservice}` is not declared; set `{HDFS_NAME_NODE}.{nameservice}`, or `{HDFS_NAME_NODE}` for a single cluster" - )) - })?, - } + hdfs_native_nameservice(config, nameservice)?.ok_or_else(|| { + invalid(format!( + "logical nameservice `{nameservice}` is not declared; set `{HDFS_NAME_NODE}.{nameservice}`" + )) + })? } None => match (&config.name_node, hdfs_native_default_fs(config)) { (Some(name_node), _) => name_node.clone(), @@ -833,14 +830,14 @@ mod tests { let (nn, rel) = hdfs_native_effective_name_node(config, "hdfs://ns-a/x").unwrap(); assert_eq!((nn.as_str(), rel), ("hdfs://a1:8020,hdfs://a2:8020", "x")); } - // An undeclared one falls back to the plain key only when it is set. - let (nn, _) = hdfs_native_effective_name_node(&with_default, "hdfs://ns-b/x").unwrap(); + // An undeclared one is an error, with or without the plain key. + for config in [&declared_only, &with_default] { + let err = hdfs_native_effective_name_node(config, "hdfs://ns-b/x").unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-b`"), "{err}"); + } + // The plain key serves authority-less paths only. + let (nn, _) = hdfs_native_effective_name_node(&with_default, "hdfs:///x").unwrap(); assert_eq!(nn, "hdfs://d:8020"); - let err = hdfs_native_effective_name_node(&declared_only, "hdfs://ns-b/x").unwrap_err(); - assert!( - err.to_string().contains("ns-b") && err.to_string().contains(HDFS_NAME_NODE), - "{err}" - ); // A concrete authority is never redirected. let (nn, _) = hdfs_native_effective_name_node(&with_default, "hdfs://other:9000/x").unwrap(); @@ -865,28 +862,23 @@ mod tests { ); let operators = HdfsNativeOperatorCache::default(); - for path in [ - "hdfs://ns-a/x", - "hdfs://ns-b/y", - "hdfs://ns-c/z", - "hdfs:///w", - ] { + for path in ["hdfs://ns-a/x", "hdfs://ns-b/y", "hdfs:///w"] { hdfs_native_create_operator(path, &config, &operators) .await .unwrap(); } - - // ns-a, ns-b, and the default serving both the undeclared ns-c and - // the authority-less path. + // ns-a, ns-b, and the plain key for the authority-less path; an + // undeclared nameservice never reaches an operator. assert_eq!(operators.len(), 3); - assert_eq!( - hdfs_native_batch_key(&config, "hdfs://ns-c/z"), - "hdfs://d:8020" - ); assert_eq!( hdfs_native_batch_key(&config, "hdfs://ns-b/y"), "hdfs://b:8020" ); + let err = hdfs_native_create_operator("hdfs://ns-c/z", &config, &operators) + .await + .unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-c`"), "{err}"); + assert_eq!(operators.len(), 3); } #[test] @@ -910,12 +902,12 @@ mod tests { hdfs_native_effective_name_node(config, "hdfs://other:9000/a/b").unwrap(); assert_eq!((nn.as_str(), rel), ("hdfs://other:9000", "a/b")); } - // A logical nameservice and an authority-less path use the configured - // list, ahead of `hdfs.host`. - for path in ["hdfs://ns-a/x", "hdfs:///y"] { - let (nn, _) = hdfs_native_effective_name_node(&configured, path).unwrap(); - assert_eq!(nn, "hdfs://nn1:8020,hdfs://nn2:8020"); - } + // An authority-less path uses the configured list, ahead of `hdfs.host`. + let (nn, _) = hdfs_native_effective_name_node(&configured, "hdfs:///y").unwrap(); + assert_eq!(nn, "hdfs://nn1:8020,hdfs://nn2:8020"); + // A logical nameservice never falls back to it: it must be declared. + let err = hdfs_native_effective_name_node(&configured, "hdfs://ns-a/x").unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-a`"), "{err}"); // Only an authority-less path falls back to `fs.defaultFS`. let (nn, rel) = hdfs_native_effective_name_node(&default_fs, "hdfs:///y").unwrap(); assert_eq!((nn.as_str(), rel), ("hdfs://nn:8020", "y")); @@ -1026,7 +1018,7 @@ mod tests { // run under tokio; `test_hdfs_native_operator_build_needs_no_runtime` // pins that the build itself does not. #[tokio::test] - async fn test_hdfs_native_create_operator_configured_name_node_serves_logical_paths() { + async fn test_hdfs_native_create_operator_plain_key_serves_authority_less_paths() { let config = Arc::new( hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), @@ -1036,11 +1028,8 @@ mod tests { ); let operators = HdfsNativeOperatorCache::default(); - // Logical and authority-less paths share the configured list's operator. - let (_, rel) = hdfs_native_create_operator("hdfs://ns-a/a/b", &config, &operators) - .await - .unwrap(); - hdfs_native_create_operator("hdfs:///c", &config, &operators) + // Authority-less paths share the configured list's operator. + let (_, rel) = hdfs_native_create_operator("hdfs:///a/b", &config, &operators) .await .unwrap(); assert_eq!(rel, "a/b"); @@ -1052,7 +1041,12 @@ mod tests { .is_some() ); - // A concrete authority is another cluster, never the configured one. + // A logical nameservice must be declared; a concrete authority is + // another cluster, never the configured one. + let err = hdfs_native_create_operator("hdfs://ns-a/c", &config, &operators) + .await + .unwrap_err(); + assert!(err.to_string().contains("`hdfs.name-node.ns-a`"), "{err}"); hdfs_native_create_operator("hdfs://other:9000/d", &config, &operators) .await .unwrap(); diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 20b94ba63a..d9fd0bd55d 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1260,13 +1260,16 @@ mod tests { "warehouse/db/t" ); - // A logical nameservice resolves only through `hdfs.name-node`. + // A logical nameservice resolves only through its declaration. assert!( storage .relativize_path("hdfs://nameservice1/a/b.parquet") .is_err() ); - let props = HashMap::from([(HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string())]); + let props = HashMap::from([( + format!("{HDFS_NAME_NODE}.nameservice1"), + "hdfs://nn:8020".to_string(), + )]); let configured = OpenDalStorage::HdfsNative(HdfsNativeStorage::new( hdfs_native_config_parse(props).unwrap(), OpenDalClientConfig::default(), diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index 1c0c9e7360..94bd5be8d3 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -114,17 +114,20 @@ mod tests { ); } - /// The HA flow: table locations carry a logical authority while - /// `hdfs.name-node` carries the (comma-separated) endpoints; it wins. + /// The HA flow: table locations carry a logical authority, and + /// `hdfs.name-node.` declares its (comma-separated) endpoints. #[tokio::test] async fn test_file_io_hdfs_configured_name_node() { require_hdfs!(); set_up(); let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) - .with_prop(HDFS_NAME_NODE, get_hdfs_endpoint()) + .with_prop( + format!("{HDFS_NAME_NODE}.logical-nameservice"), + get_hdfs_endpoint(), + ) .build(); - // The path authority is a logical name; the configured NameNode wins. + // The path authority is a logical name resolved by its declaration. let path = format!( "hdfs://logical-nameservice/{}", normalize_test_name_with_parts!("test_file_io_hdfs_configured_name_node") From 5870769e7828b8fda367e7b2edccb07b566362b3 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 22:45:36 +0300 Subject: [PATCH 30/51] fix(storage): reject userinfo and port 0 in HDFS path authorities Userinfo was silently dropped and port 0 was misread as a logical nameservice; both are DataInvalid now, and the userinfo message names the host only so a password never reaches a log. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 9c6005b37a..989f89b1df 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -231,6 +231,22 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str format!("Invalid hdfs path: {path}, expected scheme `hdfs://`"), )); }; + // Userinfo has nowhere to go and port 0 cannot be dialed; silently + // dropping the one or reading the other as a logical name would mislead. + if !url.username().is_empty() || url.password().is_some() { + // Not echoing the path: it may carry a password. + let host = url.host_str().unwrap_or_default(); + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path for host `{host}`: userinfo is not supported"), + )); + } + if url.port() == Some(0) { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path: {path}, port 0 cannot be dialed"), + )); + } let name_node = url.host_str().filter(|h| !h.is_empty()).map(|host| { url.port() @@ -943,6 +959,24 @@ mod tests { assert_eq!(rel, "a/b"); } + #[test] + fn test_hdfs_native_parse_path_rejects_userinfo_and_port_zero() { + for (path, reason) in [ + ("hdfs://alice@nn:8020/a", "userinfo"), + ("hdfs://alice:secret@nn:8020/a", "userinfo"), + ("hdfs://nn:0/a", "port 0"), + ] { + let err = hdfs_native_parse_path(path).unwrap_err().to_string(); + assert!(err.contains(reason), "{path}: {err}"); + } + // Unresolvable paths keep keying on themselves for batching. + let config = HdfsNativeConfig::default(); + assert_eq!( + hdfs_native_batch_key(&config, "hdfs://nn:0/a"), + "hdfs://nn:0/a" + ); + } + #[test] fn test_hdfs_native_parse_path_with_authority_no_path() { let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1").unwrap(); From b72d795e8ebbe3b2f0d2040f2b6bddd0bf0b2e12 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 22:48:25 +0300 Subject: [PATCH 31/51] fix(storage): validate the NameNode composed from hdfs.host and hdfs.port Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 27 ++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 989f89b1df..af47411259 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -192,15 +192,26 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result match host { Some(host) => { // An IPv6 literal needs brackets in a URI authority. - let host = if host.contains(':') && !host.starts_with('[') { + let bracketed = if host.contains(':') && !host.starts_with('[') { format!("[{host}]") } else { - host + host.clone() }; let port = port.unwrap_or(HDFS_DEFAULT_PORT); + // Validated like every other NameNode spelling, so a host that + // carries a scheme or port fails here and not at the first I/O. + let default_fs = hdfs_native_name_node(&format!("hdfs://{bracketed}:{port}")) + .ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{HDFS_HOST}`/`{HDFS_PORT}`: {host}:{port}, expected a host name or IP and a port" + ), + ) + })?; options .entry(FS_DEFAULT_FS.to_string()) - .or_insert_with(|| format!("hdfs://{host}:{port}")); + .or_insert(default_fs); } None if port.is_some() => { tracing::warn!("`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored"); @@ -645,6 +656,16 @@ mod tests { let err = parse(props).unwrap_err(); assert!(err.to_string().contains(HDFS_PORT), "{props:?}: {err}"); } + // The composed value is validated like any other NameNode spelling. + for props in [ + &[(HDFS_HOST, "nn:8020")][..], + &[(HDFS_HOST, "hdfs://nn")][..], + &[(HDFS_HOST, "nn"), (HDFS_PORT, "0")][..], + &[(HDFS_HOST, "nn/path")][..], + ] { + let err = parse(props).unwrap_err(); + assert!(err.to_string().contains(HDFS_HOST), "{props:?}: {err}"); + } } #[test] From 7f2dc7ffae73f0c7e5dc060ba9e6a410c4fdb07a Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 22:54:10 +0300 Subject: [PATCH 32/51] test(storage): use an authority-less path in the cross-runtime HDFS test A logical authority now requires a declaration, so the test must take the plain-key route. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/tests/hdfs_native_runtime_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs index 2d9cb205f2..1de2869b5b 100644 --- a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs +++ b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs @@ -46,7 +46,7 @@ mod tests { fn stat(runtime: &tokio::runtime::Runtime, file_io: &FileIO) { runtime.block_on(async { - let input = file_io.new_input("hdfs://x/f").unwrap(); + let input = file_io.new_input("hdfs:///f").unwrap(); // The fake NameNode never answers, so this fails; only the dial matters. let _ = tokio::time::timeout(Duration::from_secs(10), input.metadata()).await; }); From b9624086d7282f2c960d8c89327471a684d9ebc2 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Tue, 6 Oct 2026 23:17:24 +0300 Subject: [PATCH 33/51] test(storage): tighten the HDFS tests after review Drop three redundant unit tests, assert the warnings and the password redaction directly, make the serde round trip populate the cache first, check which error each negative case produces, cover the plain key and hdfs.host serving authority-less paths against the fixture, and name the declared-nameservice test for what it does. Co-Authored-By: Claude Fable 5.1 --- crates/storage/opendal/src/hdfs_native.rs | 199 ++++++++---------- crates/storage/opendal/src/lib.rs | 87 ++++---- .../opendal/tests/file_io_hdfs_test.rs | 68 +++++- 3 files changed, 182 insertions(+), 172 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index af47411259..12f9b3d784 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -85,8 +85,18 @@ fn hdfs_native_name_node_list(property: &str, value: &str) -> Result .collect() } -/// Parse iceberg properties to [`HdfsNativeConfig`]. -pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result { +/// Parse iceberg properties to [`HdfsNativeConfig`]; dropped properties are +/// logged as warnings. +pub(crate) fn hdfs_native_config_parse(m: HashMap) -> Result { + hdfs_native_config_parse_with(m, |warning| tracing::warn!("{warning}")) +} + +/// The parser proper, with the warning sink injected so tests can see what +/// was dropped without a tracing subscriber. +fn hdfs_native_config_parse_with( + mut m: HashMap, + mut warn: impl FnMut(String), +) -> Result { let mut cfg = HdfsNativeConfig::default(); // Entries are trimmed one by one: opendal splits the list on `,` as is, @@ -135,7 +145,9 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result // silently; the client reads `HADOOP_USER_NAME` and the Kerberos cache. for key in HDFS_UNSUPPORTED_KEYS { if m.remove(key).is_some() { - tracing::warn!("`{key}` is not supported by the hdfs-native backend and is ignored"); + warn(format!( + "`{key}` is not supported by the hdfs-native backend and is ignored" + )); } } if m.contains_key(HDFS_HADOOP_CONF_PREFIX) { @@ -214,7 +226,9 @@ pub(crate) fn hdfs_native_config_parse(mut m: HashMap) -> Result .or_insert(default_fs); } None if port.is_some() => { - tracing::warn!("`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored"); + warn(format!( + "`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored" + )); } None => {} } @@ -595,15 +609,21 @@ mod tests { assert_eq!(parse(""), None); assert_eq!(parse(" "), None); assert_eq!(parse(" , "), None); - assert_eq!( - parse(" hdfs://nn:8020/ ").as_deref(), - Some("hdfs://nn:8020") - ); - // Per entry: a space after the comma would break failover to nn2. - assert_eq!( - parse("hdfs://nn1:8020/, hdfs://nn2:8020/,").as_deref(), - Some("hdfs://nn1:8020,hdfs://nn2:8020") - ); + // `hdfs://` is optional (Hadoop's own rpc-address format is bare) and + // every entry is normalized to it, so it keys the cache like a path + // authority does; a space after a comma would otherwise break + // failover to the entry behind it. + for (value, normalized) in [ + (" hdfs://nn:8020/ ", "hdfs://nn:8020"), + ("nn:8020", "hdfs://nn:8020"), + ("[::1]:8020", "hdfs://[::1]:8020"), + ( + "hdfs://nn1:8020/, nn2:8020,", + "hdfs://nn1:8020,hdfs://nn2:8020", + ), + ] { + assert_eq!(parse(value).as_deref(), Some(normalized), "{value}"); + } } #[test] @@ -669,29 +689,43 @@ mod tests { } #[test] - fn test_hdfs_native_config_parse_rejects_bare_prefix_and_drops_unsupported_keys() { + fn test_hdfs_native_config_parse_rejects_bare_prefix() { let err = hdfs_native_config_parse(HashMap::from([( HDFS_HADOOP_CONF_PREFIX.to_string(), "x".to_string(), )])) .unwrap_err(); assert!(err.to_string().contains(HDFS_HADOOP_CONF_PREFIX), "{err}"); + } - // PyIceberg's identity keys are ignored (with a warning), never - // forwarded as Hadoop options. - let cfg = hdfs_native_config_parse(HashMap::from([ - ("hdfs.user".to_string(), "alice".to_string()), - ( - "hdfs.kerberos_ticket".to_string(), - "/tmp/krb5cc".to_string(), - ), - ])) + /// PyIceberg's identity keys and a port without a host are dropped, but + /// visibly: each leaves a warning naming the key, and none is forwarded. + #[test] + fn test_hdfs_native_config_parse_warns_about_ignored_keys() { + let mut warnings = Vec::new(); + let cfg = hdfs_native_config_parse_with( + HashMap::from([ + ("hdfs.user".to_string(), "alice".to_string()), + ( + "hdfs.kerberos_ticket".to_string(), + "/tmp/krb5cc".to_string(), + ), + (HDFS_PORT.to_string(), "9000".to_string()), + ]), + |warning| warnings.push(warning), + ) .unwrap(); - assert_eq!(cfg.options, None); + + assert_eq!(cfg.options, None, "ignored keys must not be forwarded"); + assert_eq!(warnings, [ + "`hdfs.user` is not supported by the hdfs-native backend and is ignored", + "`hdfs.kerberos_ticket` is not supported by the hdfs-native backend and is ignored", + "`hdfs.port` has no effect without `hdfs.host` and is ignored", + ]); } #[test] - fn test_hdfs_native_config_parse_name_node_requires_port() { + fn test_hdfs_native_config_parse_rejects_invalid_name_node_entries() { let parse = |value: &str| { hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), @@ -699,20 +733,8 @@ mod tests { )])) }; - // `hdfs://` is optional (Hadoop's own rpc-address format is bare) and - // every entry is normalized to it, so it keys the cache like a path - // authority does. - for (value, normalized) in [ - ("nn:8020", "hdfs://nn:8020"), - ("hdfs://nn:8020/", "hdfs://nn:8020"), - ("[::1]:8020", "hdfs://[::1]:8020"), - ( - "hdfs://nn1:8020, nn2:8020", - "hdfs://nn1:8020,hdfs://nn2:8020", - ), - ] { - assert_eq!(parse(value).unwrap().name_node.as_deref(), Some(normalized)); - } + // Logical names, other schemes, userinfo, paths, port 0, unbracketed + // IPv6 and malformed ports: every entry must be a dialable host:port. for value in [ "hdfs://ns1", "nn", @@ -745,15 +767,18 @@ mod tests { let (nn, _) = hdfs_native_effective_name_node(&parse("hdfs://nn:8020/"), "hdfs:///a").unwrap(); assert_eq!(nn, "hdfs://nn:8020"); - for default_fs in ["viewfs://cluster/", "hdfs://", ""] { - assert!(hdfs_native_effective_name_node(&parse(default_fs), "hdfs:///a").is_err()); + // Not an HDFS `host:port`, including the portless logical-name form, + // which only a declaration resolves. + for default_fs in ["viewfs://cluster/", "hdfs://", "hdfs://ns1"] { + let err = hdfs_native_effective_name_node(&parse(default_fs), "hdfs:///a").unwrap_err(); + assert!( + err.to_string().contains(FS_DEFAULT_FS) && err.to_string().contains(HDFS_NAME_NODE), + "{default_fs}: {err}" + ); } - // Portless: a logical nameservice, which only `hdfs.name-node` resolves. - let err = hdfs_native_effective_name_node(&parse("hdfs://ns1"), "hdfs:///a").unwrap_err(); - assert!( - err.to_string().contains(FS_DEFAULT_FS) && err.to_string().contains(HDFS_NAME_NODE), - "{err}" - ); + // Empty is unset, so the authority-less path has nothing to resolve to. + let err = hdfs_native_effective_name_node(&parse(""), "hdfs:///a").unwrap_err(); + assert!(err.to_string().contains(HDFS_HOST), "{err}"); // Normalized like every other source. let (nn, _) = hdfs_native_effective_name_node(&parse("nn:9000"), "hdfs:///a").unwrap(); assert_eq!(nn, "hdfs://nn:9000"); @@ -989,13 +1014,9 @@ mod tests { ] { let err = hdfs_native_parse_path(path).unwrap_err().to_string(); assert!(err.contains(reason), "{path}: {err}"); + // A password must never reach a log. + assert!(!err.contains("alice") && !err.contains("secret"), "{err}"); } - // Unresolvable paths keep keying on themselves for batching. - let config = HdfsNativeConfig::default(); - assert_eq!( - hdfs_native_batch_key(&config, "hdfs://nn:0/a"), - "hdfs://nn:0/a" - ); } #[test] @@ -1031,9 +1052,13 @@ mod tests { #[test] fn test_hdfs_native_parse_path_invalid_url_errors() { - let err = hdfs_native_parse_path("not-a-url").unwrap_err(); + let err = hdfs_native_parse_path("not-a-url").unwrap_err().to_string(); - assert!(err.to_string().contains("Invalid hdfs path")); + // The URL parser's reason is forwarded; this is not the scheme check. + assert!( + err.contains("Invalid hdfs path") && !err.contains("expected scheme"), + "{err}" + ); } #[test] @@ -1065,13 +1090,13 @@ mod tests { let config = HdfsNativeConfig::default(); // Unresolvable paths must not collapse onto a shared "" key. - assert_eq!(hdfs_native_batch_key(&config, "not-a-url"), "not-a-url"); - assert_eq!(hdfs_native_batch_key(&config, "hdfs:///a"), "hdfs:///a"); + for path in ["not-a-url", "hdfs:///a", "hdfs://nn:0/a"] { + assert_eq!(hdfs_native_batch_key(&config, path), path); + } } // Creating an operator offloads the build to a blocking thread, so these - // run under tokio; `test_hdfs_native_operator_build_needs_no_runtime` - // pins that the build itself does not. + // run under tokio. #[tokio::test] async fn test_hdfs_native_create_operator_plain_key_serves_authority_less_paths() { let config = Arc::new( @@ -1109,19 +1134,6 @@ mod tests { assert!(operators.get("hdfs://other:9000").unwrap().is_some()); } - #[tokio::test] - async fn test_hdfs_native_create_operator_uses_path_authority() { - let config = Arc::new(HdfsNativeConfig::default()); - let operators = HdfsNativeOperatorCache::default(); - - let (_, rel) = hdfs_native_create_operator("hdfs://nn:8020/a/b", &config, &operators) - .await - .unwrap(); - - assert_eq!(rel, "a/b"); - assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); - } - #[tokio::test] async fn test_hdfs_native_create_operator_caches_per_name_node() { let config = Arc::new(HdfsNativeConfig::default()); @@ -1140,35 +1152,6 @@ mod tests { assert_eq!(operators.len(), 2); } - #[tokio::test(flavor = "multi_thread")] - async fn test_hdfs_native_create_operator_concurrent_callers_share_one_entry() { - let config = Arc::new(HdfsNativeConfig::default()); - let operators = HdfsNativeOperatorCache::default(); - - let callers: Vec<_> = (0..8) - .map(|_| { - let (config, operators) = (config.clone(), operators.clone()); - tokio::spawn(async move { - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) - .await - .unwrap(); - }) - }) - .collect(); - for caller in callers { - caller.await.unwrap(); - } - - assert_eq!(operators.len(), 1); - } - - /// The build itself needs no runtime; only creating a cached operator does. - #[test] - fn test_hdfs_native_operator_build_needs_no_runtime() { - let config = HdfsNativeConfig::default(); - hdfs_native_operator_build(&config, "hdfs://nn:8020").unwrap(); - } - #[test] fn test_hdfs_native_create_operator_without_runtime_errors() { let config = Arc::new(HdfsNativeConfig::default()); @@ -1265,7 +1248,7 @@ mod tests { hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) .await .unwrap(); - assert_eq!(metrics.num_alive_tasks(), baseline + 1); + assert!(metrics.num_alive_tasks() > baseline, "no sentinel task"); // Dropping the cache aborts the sentinel; the abort lands once // the runtime schedules the task. @@ -1278,16 +1261,4 @@ mod tests { }); } } - - #[tokio::test] - async fn test_hdfs_native_create_operator_authority_less_without_config_errors() { - let config = Arc::new(HdfsNativeConfig::default()); - let operators = HdfsNativeOperatorCache::default(); - - let err = hdfs_native_create_operator("hdfs:///a/b", &config, &operators) - .await - .unwrap_err(); - - assert!(err.to_string().contains(HDFS_NAME_NODE)); - } } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index d9fd0bd55d..71377854fd 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1213,8 +1213,8 @@ mod tests { /// The configuration round-trips through serde; the operator cache does /// not and starts empty. #[cfg(feature = "opendal-hdfs-native")] - #[test] - fn test_hdfs_native_storage_serde_round_trip() { + #[tokio::test] + async fn test_hdfs_native_storage_serde_round_trip() { use iceberg::io::HDFS_NAME_NODE; let props = HashMap::from([ @@ -1228,6 +1228,15 @@ mod tests { hdfs_native_config_parse(props).unwrap(), client_config("45000").unwrap(), )); + // Populate the cache so that its absence after the round trip means + // something. + let OpenDalStorage::HdfsNative(hdfs) = &storage else { + panic!("expected the HdfsNative variant"); + }; + hdfs_native_create_operator("hdfs:///x", &hdfs.config, &hdfs.operators) + .await + .unwrap(); + assert!(hdfs.operators.get("hdfs://nn:8020").unwrap().is_some()); let value = serde_json::to_value(&storage).unwrap(); assert!(value["HdfsNative"].get("operators").is_none()); @@ -1247,64 +1256,46 @@ mod tests { assert!(restored.operators.get("hdfs://nn:8020").unwrap().is_none()); } + /// `relativize_path` follows the same resolution rule as `create_operator`. #[cfg(feature = "opendal-hdfs-native")] #[test] fn test_relativize_path_hdfs_native() { use iceberg::io::HDFS_NAME_NODE; - let storage = hdfs_native_test_storage(); + let build = |key: String| { + let props = HashMap::from([(key, "hdfs://nn:8020".to_string())]); + OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + OpenDalClientConfig::default(), + )) + }; + let unconfigured = hdfs_native_test_storage(); + let declared = build(format!("{HDFS_NAME_NODE}.nameservice1")); + let plain = build(HDFS_NAME_NODE.to_string()); + + // A concrete authority needs nothing configured. assert_eq!( - storage + unconfigured .relativize_path("hdfs://nn:8020/warehouse/db/t") .unwrap(), "warehouse/db/t" ); - - // A logical nameservice resolves only through its declaration. + // A logical nameservice needs its declaration, an authority-less path + // the plain key. + for (storage, path, rel) in [ + (&declared, "hdfs://nameservice1/a/b.parquet", "a/b.parquet"), + (&plain, "hdfs:///a/b", "a/b"), + ] { + let err = unconfigured.relativize_path(path).unwrap_err(); + assert!(err.to_string().contains(HDFS_NAME_NODE), "{err}"); + assert_eq!(storage.relativize_path(path).unwrap(), rel); + } + // Another scheme is rejected before any resolution. + let err = unconfigured.relativize_path("s3://bucket/x").unwrap_err(); assert!( - storage - .relativize_path("hdfs://nameservice1/a/b.parquet") - .is_err() + err.to_string().contains("expected scheme `hdfs://`"), + "{err}" ); - let props = HashMap::from([( - format!("{HDFS_NAME_NODE}.nameservice1"), - "hdfs://nn:8020".to_string(), - )]); - let configured = OpenDalStorage::HdfsNative(HdfsNativeStorage::new( - hdfs_native_config_parse(props).unwrap(), - OpenDalClientConfig::default(), - )); - assert_eq!( - configured - .relativize_path("hdfs://nameservice1/a/b.parquet") - .unwrap(), - "a/b.parquet" - ); - } - - #[cfg(feature = "opendal-hdfs-native")] - #[test] - fn test_relativize_path_hdfs_native_authority_less() { - use iceberg::io::HDFS_NAME_NODE; - - // Same rule as `create_operator`: usable only with `hdfs.name-node` set. - let storage = hdfs_native_test_storage(); - assert!(storage.relativize_path("hdfs:///a/b").is_err()); - - let props = HashMap::from([(HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string())]); - let storage = OpenDalStorage::HdfsNative(HdfsNativeStorage::new( - hdfs_native_config_parse(props).unwrap(), - OpenDalClientConfig::default(), - )); - assert_eq!(storage.relativize_path("hdfs:///a/b").unwrap(), "a/b"); - } - - #[cfg(feature = "opendal-hdfs-native")] - #[test] - fn test_relativize_path_hdfs_native_wrong_scheme_errors() { - let storage = hdfs_native_test_storage(); - - assert!(storage.relativize_path("s3://bucket/x").is_err()); } #[cfg(feature = "opendal-azdls")] diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index 94bd5be8d3..0871cd46c8 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -35,7 +35,7 @@ mod tests { use bytes::Bytes; use futures::StreamExt; - use iceberg::io::{FileIO, FileIOBuilder, HDFS_NAME_NODE}; + use iceberg::io::{FileIO, FileIOBuilder, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; use iceberg_storage_opendal::{OpenDalResolvingStorageFactory, OpenDalStorageFactory}; use iceberg_test_utils::{ ENV_HDFS_ENDPOINT, get_hdfs_endpoint, normalize_test_name_with_parts, set_up, @@ -88,9 +88,18 @@ mod tests { async fn test_file_io_hdfs_exists() { require_hdfs!(); let file_io = get_file_io(); + let dir = test_path("test_file_io_hdfs_exists"); + let present = format!("{dir}/present"); + let _ = file_io.delete(&present).await; + file_io + .new_output(&present) + .unwrap() + .write(Bytes::from_static(b"x")) + .await + .unwrap(); - let absent = test_path("test_file_io_hdfs_exists_absent"); - assert!(!file_io.exists(&absent).await.unwrap()); + assert!(file_io.exists(&present).await.unwrap()); + assert!(!file_io.exists(&format!("{dir}/absent")).await.unwrap()); } #[tokio::test] @@ -115,22 +124,24 @@ mod tests { } /// The HA flow: table locations carry a logical authority, and - /// `hdfs.name-node.` declares its (comma-separated) endpoints. + /// `hdfs.name-node.` declares its comma-separated endpoints + /// (the single-node fixture is listed twice to drive the list path). #[tokio::test] - async fn test_file_io_hdfs_configured_name_node() { + async fn test_file_io_hdfs_declared_nameservice() { require_hdfs!(); set_up(); + let endpoint = get_hdfs_endpoint(); let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop( format!("{HDFS_NAME_NODE}.logical-nameservice"), - get_hdfs_endpoint(), + format!("{endpoint},{endpoint}"), ) .build(); // The path authority is a logical name resolved by its declaration. let path = format!( "hdfs://logical-nameservice/{}", - normalize_test_name_with_parts!("test_file_io_hdfs_configured_name_node") + normalize_test_name_with_parts!("test_file_io_hdfs_declared_nameservice") ); let _ = file_io.delete(&path).await; @@ -148,6 +159,42 @@ mod tests { ); } + /// Authority-less paths resolve through plain `hdfs.name-node`, or through + /// PyIceberg's `hdfs.host`/`hdfs.port`; both must reach the same cluster. + #[tokio::test] + async fn test_file_io_hdfs_authority_less_paths() { + require_hdfs!(); + set_up(); + let endpoint = get_hdfs_endpoint(); + let url = url::Url::parse(&endpoint).unwrap(); + let by_name_node = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) + .with_prop(HDFS_NAME_NODE, &endpoint) + .build(); + let by_host_port = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) + .with_prop(HDFS_HOST, url.host_str().unwrap()) + .with_prop(HDFS_PORT, url.port().unwrap().to_string()) + .build(); + + let path = format!( + "hdfs:///{}", + normalize_test_name_with_parts!("test_file_io_hdfs_authority_less_paths") + ); + let _ = by_name_node.delete(&path).await; + by_name_node + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"authority-less")) + .await + .unwrap(); + + assert_eq!( + by_host_port.new_input(&path).unwrap().read().await.unwrap(), + Bytes::from_static(b"authority-less") + ); + by_host_port.delete(&path).await.unwrap(); + assert!(!by_name_node.exists(&path).await.unwrap()); + } + #[tokio::test] async fn test_file_io_hdfs_overwrite() { require_hdfs!(); @@ -198,8 +245,9 @@ mod tests { } /// Paths are batched per effective NameNode; two spellings of the fixture's - /// NameNode drive two batches through one stream. The fixture is one - /// cluster, so the keying itself is pinned by the unit tests. + /// NameNode drive two operators and two deleters through one stream. The + /// fixture is one cluster, so the keying itself is pinned by the unit + /// tests; this proves the multi-operator path completes and cleans up. #[tokio::test] async fn test_file_io_hdfs_delete_stream_two_name_nodes() { require_hdfs!(); @@ -230,7 +278,7 @@ mod tests { let stream = futures::stream::iter(paths.clone()).boxed(); file_io.delete_stream(stream).await.unwrap(); - // Both batches ran. + // Everything is gone, through whichever deleter. for path in [ format!("{dir}/a"), format!("{dir}/b"), From 1276d2d1b8fdb7ca2bd68b9b7b3f65d9e5ffa6f4 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Thu, 8 Oct 2026 11:45:43 +0300 Subject: [PATCH 34/51] fix(storage): mask userinfo in HDFS errors that echo a path or NameNode A path that failed to parse, or had another scheme, was echoed whole, so `hdfs://user:pw@nn:bad/x` put the password in the error. The NameNode config errors (`hdfs.name-node`, `hdfs.name-node.`, `hdfs.host`/`hdfs.port`, `fs.defaultFS`) echoed their values the same way. Every `Invalid hdfs path` error now goes through one helper, and the config errors use the same masking: everything between the scheme and the last `@` becomes `***`, so a password holding a raw `/`, `?`, `#` or `://` is covered too. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 128 +++++++++++++++++----- 1 file changed, 99 insertions(+), 29 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 12f9b3d784..0d58bda087 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -77,7 +77,8 @@ fn hdfs_native_name_node_list(property: &str, value: &str) -> Result Error::new( ErrorKind::DataInvalid, format!( - "Invalid `{property}` entry: {entry}, expected host:port (hdfs:// optional)" + "Invalid `{property}` entry: {}, expected host:port (hdfs:// optional)", + hdfs_native_redact(entry) ), ) }) @@ -217,7 +218,8 @@ fn hdfs_native_config_parse_with( Error::new( ErrorKind::DataInvalid, format!( - "Invalid `{HDFS_HOST}`/`{HDFS_PORT}`: {host}:{port}, expected a host name or IP and a port" + "Invalid `{HDFS_HOST}`/`{HDFS_PORT}`: {}, expected a host name or IP and a port", + hdfs_native_redact(&format!("{host}:{port}")) ), ) })?; @@ -239,38 +241,49 @@ fn hdfs_native_config_parse_with( Ok(cfg) } +/// `s` with everything between its scheme and its last `@` masked, so an +/// error message never echoes userinfo, even a password holding a raw `/`, +/// `?`, `#` or `://` that a URL parser would split elsewhere. An `@` further +/// along the path is masked the same way, which only costs context. +fn hdfs_native_redact(s: &str) -> String { + let start = s + .find("://") + .filter(|&i| { + s[..i] + .chars() + .all(|c| c.is_ascii_alphanumeric() || "+-.".contains(c)) + }) + .map_or(0, |i| i + 3); + match s[start..].rfind('@') { + Some(at) => format!("{}***{}", &s[..start], &s[start + at..]), + None => s.to_string(), + } +} + +/// A `DataInvalid` error for an HDFS path, its userinfo masked. +fn hdfs_native_invalid_path(path: &str, reason: impl std::fmt::Display) -> Error { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid hdfs path: {}, {reason}", hdfs_native_redact(path)), + ) +} + /// Parse an HDFS path into `Some("hdfs://")` (`None` when /// authority-less) and the relative path (no leading `/`, opendal style). pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str)> { - let url = Url::parse(path).map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid hdfs path: {path}: {e}"), - ) - })?; + let url = Url::parse(path).map_err(|e| hdfs_native_invalid_path(path, e))?; // Non-special schemes parse even without `//` (e.g. `hdfs:x` is a valid // non-hierarchical URL), so require the literal prefix before slicing. let (Some(after_scheme), "hdfs") = (path.strip_prefix("hdfs://"), url.scheme()) else { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid hdfs path: {path}, expected scheme `hdfs://`"), - )); + return Err(hdfs_native_invalid_path(path, "expected scheme `hdfs://`")); }; // Userinfo has nowhere to go and port 0 cannot be dialed; silently // dropping the one or reading the other as a logical name would mislead. if !url.username().is_empty() || url.password().is_some() { - // Not echoing the path: it may carry a password. - let host = url.host_str().unwrap_or_default(); - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid hdfs path for host `{host}`: userinfo is not supported"), - )); + return Err(hdfs_native_invalid_path(path, "userinfo is not supported")); } if url.port() == Some(0) { - return Err(Error::new( - ErrorKind::DataInvalid, - format!("Invalid hdfs path: {path}, port 0 cannot be dialed"), - )); + return Err(hdfs_native_invalid_path(path, "port 0 cannot be dialed")); } let name_node = url.host_str().filter(|h| !h.is_empty()).map(|host| { @@ -302,12 +315,7 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( path: &'a str, ) -> Result<(String, &'a str)> { let (authority, relative_path) = hdfs_native_parse_path(path)?; - let invalid = |reason: String| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid hdfs path: {path}, {reason}"), - ) - }; + let invalid = |reason: String| hdfs_native_invalid_path(path, reason); let name_node = match authority { Some(authority) if hdfs_native_name_node(&authority).is_some() => authority, Some(logical) => { @@ -322,7 +330,8 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( (Some(name_node), _) => name_node.clone(), (None, Some(default_fs)) => hdfs_native_name_node(default_fs).ok_or_else(|| { invalid(format!( - "`{FS_DEFAULT_FS}` {default_fs} is not an HDFS host:port, a logical nameservice requires `{HDFS_NAME_NODE}`" + "`{FS_DEFAULT_FS}` {} is not an HDFS host:port, a logical nameservice requires `{HDFS_NAME_NODE}`", + hdfs_native_redact(default_fs) )) })?, (None, None) => { @@ -1019,6 +1028,67 @@ mod tests { } } + #[test] + fn test_hdfs_native_errors_mask_userinfo() { + let path_err = |path: &str| { + hdfs_native_effective_name_node(&HdfsNativeConfig::default(), path) + .unwrap_err() + .to_string() + }; + let config_err = |key: &str, value: &str| { + let props = HashMap::from([(key.to_string(), value.to_string())]); + hdfs_native_config_parse(props) + .and_then(|config| hdfs_native_effective_name_node(&config, "hdfs:///x")) + .unwrap_err() + .to_string() + }; + // Every error that echoes a path or a NameNode value, including the + // ones that fail before the userinfo check can run. + for (err, masked) in [ + ( + path_err("hdfs://alice:s3cret@nn:bad/x"), + "hdfs://***@nn:bad/x", + ), + (path_err("s3://alice:s3cret@b/x"), "s3://***@b/x"), + ( + path_err("hdfs://alice:s3cret@nn:8020/x"), + "hdfs://***@nn:8020/x", + ), + ( + config_err(HDFS_NAME_NODE, "hdfs://alice:s3cret@nn:8020"), + "hdfs://***@nn:8020", + ), + ( + config_err("hdfs.name-node.ns", "hdfs://alice:s3cret@nn:8020"), + "hdfs://***@nn:8020", + ), + (config_err(HDFS_HOST, "alice:s3cret@nn"), "***@nn:8020"), + ( + config_err("hadoop.fs.defaultFS", "hdfs://alice:s3cret@nn:8020"), + "hdfs://***@nn:8020", + ), + // A raw `/`, `#` or `://` in the password, or a second `@`, must + // not move where the masking stops. + ( + path_err("hdfs://alice:s3/cret@nn:8020/x"), + "hdfs://***@nn:8020/x", + ), + (path_err("s3://alice:12#cret@b/x"), "s3://***@b/x"), + ( + path_err("hdfs://alice@corp:s3cret@nn:bad/x"), + "hdfs://***@nn:bad/x", + ), + ( + config_err(HDFS_NAME_NODE, "hdfs://alice:s3/cret@nn:8020"), + "hdfs://***@nn:8020", + ), + (config_err(HDFS_HOST, "alice:s3://cret@nn"), "***@nn:8020"), + ] { + assert!(err.contains(masked), "{err}"); + assert!(!err.contains("alice") && !err.contains("cret"), "{err}"); + } + } + #[test] fn test_hdfs_native_parse_path_with_authority_no_path() { let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1").unwrap(); From 1fd349240b6535fe1350937f8f79d00b1270f9e0 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Thu, 8 Oct 2026 11:45:43 +0300 Subject: [PATCH 35/51] fix(storage): recover a poisoned HDFS operator cache lock A panic elsewhere while the cache lock was held turned every later HDFS call into a permanent `Unexpected` error. Writes only ever replace whole entries, so the map stays consistent: both lock sites now recover the guard with `PoisonError::into_inner`, and `get`/`insert` are infallible. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 76 ++++++++++++----------- crates/storage/opendal/src/lib.rs | 4 +- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 0d58bda087..02e34231b9 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -19,7 +19,7 @@ use std::collections::HashMap; use std::collections::hash_map::Entry; -use std::sync::{Arc, RwLock, Weak}; +use std::sync::{Arc, PoisonError, RwLock, Weak}; use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; use iceberg::{Error, ErrorKind, Result}; @@ -469,32 +469,31 @@ impl CachedOperator { } } +// A poisoned lock is recovered rather than reported: writes only ever +// replace whole entries, so a panic elsewhere cannot leave the map broken. impl HdfsNativeOperatorCache { - pub(crate) fn get(&self, name_node: &str) -> Result> { - Ok(self - .0 + pub(crate) fn get(&self, name_node: &str) -> Option { + self.0 .read() - .map_err(poisoned)? + .unwrap_or_else(PoisonError::into_inner) .get(name_node) .filter(|cached| cached.runtime_alive()) - .map(|cached| cached.operator.clone())) + .map(|cached| cached.operator.clone()) } /// Inserts `op` unless a concurrent caller got there first, returning /// whichever operator the cache now holds; a stale entry is replaced. - fn insert(&self, name_node: String, op: Operator, handle: &Handle) -> Result { - let mut operators = self.0.write().map_err(poisoned)?; + fn insert(&self, name_node: String, op: Operator, handle: &Handle) -> Operator { + let mut operators = self.0.write().unwrap_or_else(PoisonError::into_inner); match operators.entry(name_node) { - Entry::Occupied(entry) if entry.get().runtime_alive() => { - Ok(entry.get().operator.clone()) - } + Entry::Occupied(entry) if entry.get().runtime_alive() => entry.get().operator.clone(), Entry::Occupied(mut entry) => { entry.insert(CachedOperator::new(op.clone(), handle)); - Ok(op) + op } Entry::Vacant(entry) => { entry.insert(CachedOperator::new(op.clone(), handle)); - Ok(op) + op } } } @@ -505,10 +504,6 @@ impl HdfsNativeOperatorCache { } } -fn poisoned(_: T) -> Error { - Error::new(ErrorKind::Unexpected, "HDFS operator cache lock poisoned") -} - /// Creates an operator for the path, reusing the cached one for its /// effective NameNode. pub(crate) async fn hdfs_native_create_operator<'a>( @@ -518,7 +513,7 @@ pub(crate) async fn hdfs_native_create_operator<'a>( ) -> Result<(Operator, &'a str)> { let (name_node, relative_path) = hdfs_native_effective_name_node(config, path)?; - if let Some(op) = operators.get(&name_node)? { + if let Some(op) = operators.get(&name_node) { return Ok((op, relative_path)); } @@ -542,7 +537,7 @@ pub(crate) async fn hdfs_native_create_operator<'a>( .map_err(|e| { Error::new(ErrorKind::Unexpected, "HDFS operator build task failed").with_source(e) })??; - Ok((operators.insert(name_node, op, &handle)?, relative_path)) + Ok((operators.insert(name_node, op, &handle), relative_path)) } /// Returns the `delete_stream` grouping key for a path: the effective @@ -813,7 +808,7 @@ mod tests { } assert_eq!(operators.len(), 1); - assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); + assert!(operators.get("hdfs://nn:8020").is_some()); } #[test] @@ -1184,12 +1179,7 @@ mod tests { .unwrap(); assert_eq!(rel, "a/b"); assert_eq!(operators.len(), 1); - assert!( - operators - .get("hdfs://nn1:8020,hdfs://nn2:8020") - .unwrap() - .is_some() - ); + assert!(operators.get("hdfs://nn1:8020,hdfs://nn2:8020").is_some()); // A logical nameservice must be declared; a concrete authority is // another cluster, never the configured one. @@ -1201,7 +1191,7 @@ mod tests { .await .unwrap(); assert_eq!(operators.len(), 2); - assert!(operators.get("hdfs://other:9000").unwrap().is_some()); + assert!(operators.get("hdfs://other:9000").is_some()); } #[tokio::test] @@ -1250,17 +1240,31 @@ mod tests { // Two callers racing for one NameNode: the first insert wins and both // get the cached operator. - let first = operators - .insert("hdfs://nn:8020".to_string(), build("/first"), &handle) - .unwrap(); - let second = operators - .insert("hdfs://nn:8020".to_string(), build("/second"), &handle) - .unwrap(); + let first = operators.insert("hdfs://nn:8020".to_string(), build("/first"), &handle); + let second = operators.insert("hdfs://nn:8020".to_string(), build("/second"), &handle); assert_eq!(first.info().root(), "/first/"); assert_eq!(second.info().root(), "/first/"); assert_eq!(operators.len(), 1); } + #[tokio::test] + async fn test_hdfs_native_operator_cache_survives_a_poisoned_lock() { + let operators = HdfsNativeOperatorCache::default(); + // A panic elsewhere while the lock is held must not break HDFS I/O + // for good: every write replaces a whole entry, so the map is intact. + let _ = std::panic::catch_unwind(|| { + let _guard = operators.0.write().unwrap(); + panic!("poisoning the operator cache lock"); + }); + assert!(operators.0.is_poisoned()); + + let config = Arc::new(HdfsNativeConfig::default()); + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) + .await + .unwrap(); + assert!(operators.get("hdfs://nn:8020").is_some()); + } + #[test] fn test_hdfs_native_operator_cache_rebuilds_after_runtime_shutdown() { let flavors: [fn() -> tokio::runtime::Runtime; 2] = [ @@ -1281,19 +1285,19 @@ mod tests { .await .unwrap(); }); - assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); + assert!(operators.get("hdfs://nn:8020").is_some()); // The building runtime is gone: the entry is stale and the next // runtime to use it rebuilds it. drop(runtime); - assert!(operators.get("hdfs://nn:8020").unwrap().is_none()); + assert!(operators.get("hdfs://nn:8020").is_none()); let next = build_runtime(); next.block_on(async { hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) .await .unwrap(); }); - assert!(operators.get("hdfs://nn:8020").unwrap().is_some()); + assert!(operators.get("hdfs://nn:8020").is_some()); assert_eq!(operators.len(), 1); } } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index acb257e8a5..9ed8447058 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1235,7 +1235,7 @@ mod tests { hdfs_native_create_operator("hdfs:///x", &hdfs.config, &hdfs.operators) .await .unwrap(); - assert!(hdfs.operators.get("hdfs://nn:8020").unwrap().is_some()); + assert!(hdfs.operators.get("hdfs://nn:8020").is_some()); let value = serde_json::to_value(&storage).unwrap(); assert!(value["HdfsNative"].get("operators").is_none()); @@ -1252,7 +1252,7 @@ mod tests { Some(&"true".to_string()) ); assert_eq!(restored.client_config.io_timeout(), Duration::from_secs(45)); - assert!(restored.operators.get("hdfs://nn:8020").unwrap().is_none()); + assert!(restored.operators.get("hdfs://nn:8020").is_none()); } /// `relativize_path` follows the same resolution rule as `create_operator`. From 894a8eda97cc45f58582a29866b21560b5b66818 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Thu, 8 Oct 2026 11:45:43 +0300 Subject: [PATCH 36/51] test(storage): flush the fake NameNode before sampling dials The runtime test sampled its dial counter before the listener thread had necessarily counted every dial from the first runtime. It now drops that runtime and waits for the listener to close a marker connection; accepts are served in order, so every earlier dial is counted first. A failed accept no longer stops the listener. The runtime-shutdown unit test also asserts that the rebuilt operator is a new instance, not the stale one with a fresh sentinel. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 32 ++++++++++++------- .../opendal/tests/hdfs_native_runtime_test.rs | 23 ++++++++++--- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 02e34231b9..279811c006 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -1280,25 +1280,35 @@ mod tests { let config = Arc::new(HdfsNativeConfig::default()); let runtime = build_runtime(); - runtime.block_on(async { - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) - .await - .unwrap(); - }); + let (stale, _) = runtime + .block_on(hdfs_native_create_operator( + "hdfs://nn:8020/a", + &config, + &operators, + )) + .unwrap(); assert!(operators.get("hdfs://nn:8020").is_some()); // The building runtime is gone: the entry is stale and the next - // runtime to use it rebuilds it. + // runtime to use it builds a new operator in its place. drop(runtime); assert!(operators.get("hdfs://nn:8020").is_none()); let next = build_runtime(); - next.block_on(async { - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) - .await - .unwrap(); - }); + let (rebuilt, _) = next + .block_on(hdfs_native_create_operator( + "hdfs://nn:8020/a", + &config, + &operators, + )) + .unwrap(); assert!(operators.get("hdfs://nn:8020").is_some()); assert_eq!(operators.len(), 1); + let (_, stale) = stale.into_parts(); + let (_, rebuilt) = rebuilt.into_parts(); + assert!( + !Arc::ptr_eq(&stale, &rebuilt), + "the stale operator was reused" + ); } } diff --git a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs index 1de2869b5b..2235314e4d 100644 --- a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs +++ b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs @@ -21,7 +21,8 @@ #[cfg(feature = "opendal-hdfs-native")] mod tests { - use std::net::TcpListener; + use std::io::Read; + use std::net::{TcpListener, TcpStream}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -29,7 +30,8 @@ mod tests { use iceberg::io::{FileIO, FileIOBuilder, HDFS_NAME_NODE}; use iceberg_storage_opendal::OpenDalStorageFactory; - /// Accepts and immediately closes connections, counting them. + /// Accepts and immediately closes connections, counting them. A failed + /// accept (a dial reset while queued) is skipped so the listener lives on. fn fake_name_node() -> (u16, Arc) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); @@ -37,7 +39,7 @@ mod tests { let counter = dials.clone(); std::thread::spawn(move || { for stream in listener.incoming() { - let Ok(_stream) = stream else { break }; + let Ok(_stream) = stream else { continue }; counter.fetch_add(1, Ordering::SeqCst); } }); @@ -62,10 +64,21 @@ mod tests { let first = tokio::runtime::Runtime::new().unwrap(); stat(&first, &file_io); - let after_first = dials.load(Ordering::SeqCst); - assert!(after_first >= 1, "the NameNode was never dialed"); + assert!( + dials.load(Ordering::SeqCst) >= 1, + "the NameNode was never dialed" + ); drop(first); + // The listener accepts in order, so once it has closed this marker + // connection, every dial from the first runtime has been counted. + let mut marker = TcpStream::connect(("127.0.0.1", port)).unwrap(); + marker + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + assert_eq!(marker.read(&mut [0]).unwrap(), 0); + let after_first = dials.load(Ordering::SeqCst); + // Reusing the FileIO from another runtime used to panic inside // hdfs-native, whose client was bound to the dropped runtime. let second = tokio::runtime::Runtime::new().unwrap(); From 30d74a88e31734cc34c85be017f0967a663674c5 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Thu, 8 Oct 2026 11:45:43 +0300 Subject: [PATCH 37/51] docs(storage): state where portless HDFS authorities differ from Hadoop Hadoop resolves a portless authority from hdfs-site.xml, else dials port 8020. Here it must be declared with `hdfs.name-node.`, because opendal builds the client as `hdfs://nameservice` and never hands hdfs-native the authority. The `HDFS_NAME_NODE` and resolver docs no longer claim Hadoop parity, the error says to add a port or declare the nameservice, and the README notes the limitation in its own note. Co-Authored-By: Claude Opus 5.5 --- crates/iceberg/src/io/storage/config/hdfs.rs | 14 +++++++------- crates/storage/opendal/README.md | 4 +++- crates/storage/opendal/src/hdfs_native.rs | 19 +++++++++---------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs.rs index e2f1d3cfe8..ef9e16bf65 100644 --- a/crates/iceberg/src/io/storage/config/hdfs.rs +++ b/crates/iceberg/src/io/storage/config/hdfs.rs @@ -19,14 +19,14 @@ /// HDFS NameNode RPC endpoint(s) as `host:port`, e.g. `hdfs://namenode:8020`; /// a comma-separated list enables HA failover. Serves authority-less paths -/// (`hdfs:///path`) only. As in Hadoop, an authority with a port is always -/// used as is, and a logical nameservice authority (`hdfs://ns1/path`, no -/// port) must be declared: `hdfs.name-node.` takes the same -/// list and is sugar for Hadoop's `dfs.ha.namenodes.` and +/// (`hdfs:///path`) only; a path authority with a port is used as is. A +/// portless authority (`hdfs://ns1/path`) must be a nameservice declared in +/// properties: `hdfs.name-node.` takes the same list and is +/// sugar for Hadoop's `dfs.ha.namenodes.` and /// `dfs.namenode.rpc-address..`, which are honored as well -/// when passed through [`HDFS_HADOOP_CONF_PREFIX`]. An undeclared nameservice -/// is an error rather than a guess, and nameservices defined only in -/// `$HADOOP_CONF_DIR` are not visible here, so declare them in properties. +/// when passed through [`HDFS_HADOOP_CONF_PREFIX`]. Unlike Hadoop, an +/// undeclared one is an error: it is neither resolved from +/// `$HADOOP_CONF_DIR` nor dialed on the default port 8020. pub const HDFS_NAME_NODE: &str = "hdfs.name-node"; /// NameNode host for authority-less paths, as in PyIceberg; paths that carry /// an authority ignore it, as they do there. Combined with [`HDFS_PORT`] into diff --git a/crates/storage/opendal/README.md b/crates/storage/opendal/README.md index 45fa27e582..ac3a8b62ef 100644 --- a/crates/storage/opendal/README.md +++ b/crates/storage/opendal/README.md @@ -36,7 +36,9 @@ OpenDAL-based storage backend implementations for [Apache Iceberg Rust](https:// You can enable all stable storage backends at once using the `opendal-all` feature flag. -> Note that `opendal-oss`, `opendal-azdls` and `opendal-hdfs-native` are currently experimental and not included in `opendal-all`. +> Note that `opendal-oss` and `opendal-azdls` are currently experimental and not included in `opendal-all`. + +> `opendal-hdfs-native` is experimental and not included in `opendal-all`. A portless path authority such as `hdfs://nameservice1/warehouse` must be declared in properties, e.g. `hdfs.name-node.nameservice1 = nn1:8020,nn2:8020`: unlike Hadoop, nameservices defined only in `$HADOOP_CONF_DIR` are not resolved, and a portless host is not dialed on the default port 8020. ## Usage diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 279811c006..29067c9fd0 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -304,12 +304,11 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str Ok((name_node, rel)) } -/// Resolves the effective NameNode for a path, plus the relative path. As in -/// Hadoop, an authority with a port is used as is; a logical nameservice -/// authority (no port) resolves through its declaration, and an -/// authority-less path through `hdfs.name-node`, else `fs.defaultFS`. The -/// operator cache, `delete_stream` batching and `relativize_path` all go -/// through this, so they cannot drift apart. +/// Resolves the effective NameNode for a path, plus the relative path. An +/// authority with a port is used as is, a portless one must be a declared +/// nameservice, and an authority-less path uses `hdfs.name-node`, else +/// `fs.defaultFS`. The operator cache, `delete_stream` batching and +/// `relativize_path` all go through this, so they cannot drift apart. pub(crate) fn hdfs_native_effective_name_node<'a>( config: &HdfsNativeConfig, path: &'a str, @@ -318,11 +317,11 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( let invalid = |reason: String| hdfs_native_invalid_path(path, reason); let name_node = match authority { Some(authority) if hdfs_native_name_node(&authority).is_some() => authority, - Some(logical) => { - let nameservice = logical.trim_start_matches("hdfs://"); + Some(portless) => { + let nameservice = portless.trim_start_matches("hdfs://"); hdfs_native_nameservice(config, nameservice)?.ok_or_else(|| { invalid(format!( - "logical nameservice `{nameservice}` is not declared; set `{HDFS_NAME_NODE}.{nameservice}`" + "`{nameservice}` has no port and is not a declared nameservice; add the port or set `{HDFS_NAME_NODE}.{nameservice}`" )) })? } @@ -978,7 +977,7 @@ mod tests { let (nn, rel) = hdfs_native_effective_name_node(&default_fs, "hdfs:///y").unwrap(); assert_eq!((nn.as_str(), rel), ("hdfs://nn:8020", "y")); let err = hdfs_native_effective_name_node(&default_fs, "hdfs://ns-a/x").unwrap_err(); - assert!(err.to_string().contains("logical nameservice `ns-a`")); + assert!(err.to_string().contains("`hdfs.name-node.ns-a`"), "{err}"); // Nothing applicable: pointed errors. for path in ["hdfs:///a", "hdfs://ns-a/x"] { let err = hdfs_native_effective_name_node(&unconfigured, path).unwrap_err(); From 86d81b4131e7ec8da5a2828896172973454318ef Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 21:45:07 +0300 Subject: [PATCH 38/51] refactor(storage): rename the HDFS config module to hdfs_native Matches its siblings, which are named after their storage feature (azdls, gcs, hf, oss, s3). The module is private and re-exported, so the public iceberg::io::HDFS_* paths are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../iceberg/src/io/storage/config/{hdfs.rs => hdfs_native.rs} | 0 crates/iceberg/src/io/storage/config/mod.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename crates/iceberg/src/io/storage/config/{hdfs.rs => hdfs_native.rs} (100%) diff --git a/crates/iceberg/src/io/storage/config/hdfs.rs b/crates/iceberg/src/io/storage/config/hdfs_native.rs similarity index 100% rename from crates/iceberg/src/io/storage/config/hdfs.rs rename to crates/iceberg/src/io/storage/config/hdfs_native.rs diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index fe545d7a5b..8ea4af080e 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -32,7 +32,7 @@ mod azdls; mod gcs; -mod hdfs; +mod hdfs_native; mod hf; mod oss; mod s3; @@ -41,7 +41,7 @@ use std::collections::HashMap; pub use azdls::*; pub use gcs::*; -pub use hdfs::*; +pub use hdfs_native::*; pub use hf::*; pub use oss::*; pub use s3::*; From a347b847256c7579a9a5c7ca11879e52a5f107c9 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 22:06:08 +0300 Subject: [PATCH 39/51] test(storage): move the HDFS tests out of lib.rs into hdfs_native The serde round-trip and relativize_path tests now live next to the code they cover, with the module's test_hdfs_native_ prefix. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 93 ++++++++++++++++++++++ crates/storage/opendal/src/lib.rs | 96 ----------------------- 2 files changed, 93 insertions(+), 96 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 29067c9fd0..8b660f3764 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -1161,6 +1161,48 @@ mod tests { // Creating an operator offloads the build to a blocking thread, so these // run under tokio. + /// `relativize_path` follows the same resolution rule as `create_operator`. + #[test] + fn test_hdfs_native_relativize_path() { + let build = |key: String| { + let props = HashMap::from([(key, "hdfs://nn:8020".to_string())]); + crate::OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + OpenDalClientConfig::default(), + )) + }; + let unconfigured = crate::OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + HdfsNativeConfig::default(), + OpenDalClientConfig::default(), + )); + let declared = build(format!("{HDFS_NAME_NODE}.nameservice1")); + let plain = build(HDFS_NAME_NODE.to_string()); + + // A concrete authority needs nothing configured. + assert_eq!( + unconfigured + .relativize_path("hdfs://nn:8020/warehouse/db/t") + .unwrap(), + "warehouse/db/t" + ); + // A logical nameservice needs its declaration, an authority-less path + // the plain key. + for (storage, path, rel) in [ + (&declared, "hdfs://nameservice1/a/b.parquet", "a/b.parquet"), + (&plain, "hdfs:///a/b", "a/b"), + ] { + let err = unconfigured.relativize_path(path).unwrap_err(); + assert!(err.to_string().contains(HDFS_NAME_NODE), "{err}"); + assert_eq!(storage.relativize_path(path).unwrap(), rel); + } + // Another scheme is rejected before any resolution. + let err = unconfigured.relativize_path("s3://bucket/x").unwrap_err(); + assert!( + err.to_string().contains("expected scheme `hdfs://`"), + "{err}" + ); + } + #[tokio::test] async fn test_hdfs_native_create_operator_plain_key_serves_authority_less_paths() { let config = Arc::new( @@ -1344,4 +1386,55 @@ mod tests { }); } } + + /// The configuration round-trips through serde; the operator cache does + /// not and starts empty. + #[tokio::test] + async fn test_hdfs_native_storage_serde_round_trip() { + let props = HashMap::from([ + (HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string()), + ( + "hadoop.dfs.client.use.datanode.hostname".to_string(), + "true".to_string(), + ), + ]); + let storage = crate::OpenDalStorage::HdfsNative(HdfsNativeStorage::new( + hdfs_native_config_parse(props).unwrap(), + OpenDalClientConfig::from_properties(&HashMap::from([( + crate::OPENDAL_IO_TIMEOUT_MS.to_string(), + "45000".to_string(), + )])) + .unwrap(), + )); + // Populate the cache so that its absence after the round trip means + // something. + let crate::OpenDalStorage::HdfsNative(hdfs) = &storage else { + panic!("expected the HdfsNative variant"); + }; + hdfs_native_create_operator("hdfs:///x", &hdfs.config, &hdfs.operators) + .await + .unwrap(); + assert!(hdfs.operators.get("hdfs://nn:8020").is_some()); + + let value = serde_json::to_value(&storage).unwrap(); + assert!(value["HdfsNative"].get("operators").is_none()); + let crate::OpenDalStorage::HdfsNative(restored) = serde_json::from_value(value).unwrap() + else { + panic!("expected the HdfsNative variant"); + }; + assert_eq!(restored.config.name_node.as_deref(), Some("hdfs://nn:8020")); + assert_eq!( + restored + .config + .options + .as_ref() + .and_then(|o| o.get("dfs.client.use.datanode.hostname")), + Some(&"true".to_string()) + ); + assert_eq!( + restored.client_config.io_timeout(), + std::time::Duration::from_secs(45) + ); + assert!(restored.operators.get("hdfs://nn:8020").is_none()); + } } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 9ed8447058..660e64ba7d 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1201,102 +1201,6 @@ mod tests { ); } - #[cfg(feature = "opendal-hdfs-native")] - fn hdfs_native_test_storage() -> OpenDalStorage { - OpenDalStorage::HdfsNative(HdfsNativeStorage::new( - opendal::services::HdfsNativeConfig::default(), - OpenDalClientConfig::default(), - )) - } - - /// The configuration round-trips through serde; the operator cache does - /// not and starts empty. - #[cfg(feature = "opendal-hdfs-native")] - #[tokio::test] - async fn test_hdfs_native_storage_serde_round_trip() { - use iceberg::io::HDFS_NAME_NODE; - - let props = HashMap::from([ - (HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string()), - ( - "hadoop.dfs.client.use.datanode.hostname".to_string(), - "true".to_string(), - ), - ]); - let storage = OpenDalStorage::HdfsNative(HdfsNativeStorage::new( - hdfs_native_config_parse(props).unwrap(), - client_config("45000").unwrap(), - )); - // Populate the cache so that its absence after the round trip means - // something. - let OpenDalStorage::HdfsNative(hdfs) = &storage else { - panic!("expected the HdfsNative variant"); - }; - hdfs_native_create_operator("hdfs:///x", &hdfs.config, &hdfs.operators) - .await - .unwrap(); - assert!(hdfs.operators.get("hdfs://nn:8020").is_some()); - - let value = serde_json::to_value(&storage).unwrap(); - assert!(value["HdfsNative"].get("operators").is_none()); - let OpenDalStorage::HdfsNative(restored) = serde_json::from_value(value).unwrap() else { - panic!("expected the HdfsNative variant"); - }; - assert_eq!(restored.config.name_node.as_deref(), Some("hdfs://nn:8020")); - assert_eq!( - restored - .config - .options - .as_ref() - .and_then(|o| o.get("dfs.client.use.datanode.hostname")), - Some(&"true".to_string()) - ); - assert_eq!(restored.client_config.io_timeout(), Duration::from_secs(45)); - assert!(restored.operators.get("hdfs://nn:8020").is_none()); - } - - /// `relativize_path` follows the same resolution rule as `create_operator`. - #[cfg(feature = "opendal-hdfs-native")] - #[test] - fn test_relativize_path_hdfs_native() { - use iceberg::io::HDFS_NAME_NODE; - - let build = |key: String| { - let props = HashMap::from([(key, "hdfs://nn:8020".to_string())]); - OpenDalStorage::HdfsNative(HdfsNativeStorage::new( - hdfs_native_config_parse(props).unwrap(), - OpenDalClientConfig::default(), - )) - }; - let unconfigured = hdfs_native_test_storage(); - let declared = build(format!("{HDFS_NAME_NODE}.nameservice1")); - let plain = build(HDFS_NAME_NODE.to_string()); - - // A concrete authority needs nothing configured. - assert_eq!( - unconfigured - .relativize_path("hdfs://nn:8020/warehouse/db/t") - .unwrap(), - "warehouse/db/t" - ); - // A logical nameservice needs its declaration, an authority-less path - // the plain key. - for (storage, path, rel) in [ - (&declared, "hdfs://nameservice1/a/b.parquet", "a/b.parquet"), - (&plain, "hdfs:///a/b", "a/b"), - ] { - let err = unconfigured.relativize_path(path).unwrap_err(); - assert!(err.to_string().contains(HDFS_NAME_NODE), "{err}"); - assert_eq!(storage.relativize_path(path).unwrap(), rel); - } - // Another scheme is rejected before any resolution. - let err = unconfigured.relativize_path("s3://bucket/x").unwrap_err(); - assert!( - err.to_string().contains("expected scheme `hdfs://`"), - "{err}" - ); - } - #[cfg(feature = "opendal-azdls")] #[test] fn test_relativize_path_azdls() { From 39b3cc59aed63a020ef7e88ea9da0cc3982b9445 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 22:14:49 +0300 Subject: [PATCH 40/51] refactor(storage): read HDFS properties through #[derive(Properties)] hdfs.name-node, the hdfs.name-node. and hadoop. prefixes, hdfs.host and hdfs.port are declared in one struct parsed by the Properties macro instead of being looked up by hand. Host and port stay trimmed strings so a blank value still counts as unset, as in PyIceberg. Validation, warnings and error messages are unchanged. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 157 +++++++++++----------- 1 file changed, 81 insertions(+), 76 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 8b660f3764..1e22117590 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -23,6 +23,7 @@ use std::sync::{Arc, PoisonError, RwLock, Weak}; use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; use iceberg::{Error, ErrorKind, Result}; +use iceberg_property_macro::Properties; use opendal::Operator; use opendal::services::HdfsNativeConfig; use serde::{Deserialize, Serialize}; @@ -86,6 +87,44 @@ fn hdfs_native_name_node_list(property: &str, value: &str) -> Result .collect() } +/// Prefix of `hdfs.name-node.`, which declares one HA nameservice. +const HDFS_NAME_NODE_DECLARATION_PREFIX: &str = "hdfs.name-node."; + +/// The properties this backend reads; unknown keys are ignored. Blank values +/// count as unset, as in PyIceberg. +#[derive(Properties)] +struct HdfsNativeProperties { + /// `hdfs.name-node`, each entry normalized. + #[property( + key = HDFS_NAME_NODE, + default = Vec::new(), + parse_with = parse_name_node_list + )] + name_node: Vec, + /// `hdfs.name-node.` declarations, keyed by nameservice. + #[property(prefix = HDFS_NAME_NODE_DECLARATION_PREFIX)] + nameservices: HashMap, + #[property(key = HDFS_HOST, default = "", parse_with = parse_trimmed)] + host: String, + #[property(key = HDFS_PORT, default = "", parse_with = parse_trimmed)] + port: String, + /// `hadoop.*` keys with the prefix stripped, forwarded to the client. + #[property(prefix = HDFS_HADOOP_CONF_PREFIX)] + hadoop: HashMap, +} + +// Entries are trimmed one by one: opendal splits the list on `,` as is, so a +// space after a comma would break failover to that NameNode. An empty list +// stays unset because `Operator::from_config` bypasses the builder's +// empty-string guard and `Some("")` would shadow the path-authority fallback. +fn parse_name_node_list(value: &str) -> Result> { + hdfs_native_name_node_list(HDFS_NAME_NODE, value) +} + +fn parse_trimmed(value: &str) -> Result { + Ok(value.trim().to_string()) +} + /// Parse iceberg properties to [`HdfsNativeConfig`]; dropped properties are /// logged as warnings. pub(crate) fn hdfs_native_config_parse(m: HashMap) -> Result { @@ -95,40 +134,22 @@ pub(crate) fn hdfs_native_config_parse(m: HashMap) -> Result, + m: HashMap, mut warn: impl FnMut(String), ) -> Result { - let mut cfg = HdfsNativeConfig::default(); - - // Entries are trimmed one by one: opendal splits the list on `,` as is, - // so a space after a comma would break failover to that NameNode. An - // empty result is dropped because `Operator::from_config` bypasses the - // builder's empty-string guard and `Some("")` would shadow the - // path-authority fallback below. - if let Some(name_node) = m.remove(HDFS_NAME_NODE) { - let entries = hdfs_native_name_node_list(HDFS_NAME_NODE, &name_node)?; - if !entries.is_empty() { - cfg.name_node = Some(entries.join(",")); - } - } + let props = HdfsNativeProperties::from_properties(&m)?; + let mut options = props.hadoop; // `hdfs.name-node.` is sugar for Hadoop's own declaration of // an HA nameservice, which the resolver reads back from the options. - let nameservice_prefix = format!("{HDFS_NAME_NODE}."); - let declared_keys: Vec = m - .keys() - .filter(|key| key.starts_with(&nameservice_prefix)) - .cloned() - .collect(); - let mut declared = Vec::new(); - for key in declared_keys { - let value = m.remove(&key).unwrap_or_default(); - let nameservice = key[nameservice_prefix.len()..].to_string(); + // Explicit `hadoop.` keys win over the sugar. + for (nameservice, value) in props.nameservices { + let key = format!("{HDFS_NAME_NODE_DECLARATION_PREFIX}{nameservice}"); if nameservice.is_empty() || nameservice.chars().any(char::is_whitespace) { return Err(Error::new( ErrorKind::DataInvalid, format!( - "Invalid property `{key}`: a nameservice name must follow `{nameservice_prefix}`" + "Invalid property `{key}`: a nameservice name must follow `{HDFS_NAME_NODE_DECLARATION_PREFIX}`" ), )); } @@ -139,13 +160,23 @@ fn hdfs_native_config_parse_with( format!("Invalid property `{key}`: no NameNodes"), )); } - declared.push((nameservice, entries)); + let ids: Vec = (0..entries.len()).map(|i| format!("nn{i}")).collect(); + options + .entry(format!("{HA_NAMENODES_PREFIX}.{nameservice}")) + .or_insert_with(|| ids.join(",")); + for (id, entry) in ids.iter().zip(&entries) { + options + .entry(format!( + "{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}" + )) + .or_insert_with(|| entry.trim_start_matches("hdfs://").to_string()); + } } // A config carried over from PyIceberg would otherwise change identity // silently; the client reads `HADOOP_USER_NAME` and the Kerberos cache. for key in HDFS_UNSUPPORTED_KEYS { - if m.remove(key).is_some() { + if m.contains_key(key) { warn(format!( "`{key}` is not supported by the hdfs-native backend and is ignored" )); @@ -160,55 +191,31 @@ fn hdfs_native_config_parse_with( )); } - let host = m - .remove(HDFS_HOST) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()); - let port = m - .remove(HDFS_PORT) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .map(|port| { - port.parse::().map_err(|e| { - Error::new( - ErrorKind::DataInvalid, - format!("Invalid `{HDFS_PORT}`: {port}: {e}"), - ) - }) - }) - .transpose()?; - - let mut options: HashMap = m - .into_iter() - .filter_map(|(key, value)| { - key.strip_prefix(HDFS_HADOOP_CONF_PREFIX) - .map(|stripped| (stripped.to_string(), value)) - }) - .collect(); - // Explicit `hadoop.` keys win over the sugar. - for (nameservice, entries) in declared { - let ids: Vec = (0..entries.len()).map(|i| format!("nn{i}")).collect(); - options - .entry(format!("{HA_NAMENODES_PREFIX}.{nameservice}")) - .or_insert_with(|| ids.join(",")); - for (id, entry) in ids.iter().zip(&entries) { - options - .entry(format!( - "{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}" - )) - .or_insert_with(|| entry.trim_start_matches("hdfs://").to_string()); - } - } + let port = match props.port.as_str() { + "" => None, + port => Some(port.parse::().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid `{HDFS_PORT}`: {port}: {e}"), + ) + })?), + }; // PyIceberg's `hdfs.host`/`hdfs.port` name the filesystem for // authority-less paths, which is what Hadoop's `fs.defaultFS` means; an // explicit `hadoop.fs.defaultFS` wins. - match host { - Some(host) => { + match props.host.as_str() { + "" if port.is_some() => { + warn(format!( + "`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored" + )); + } + "" => {} + host => { // An IPv6 literal needs brackets in a URI authority. let bracketed = if host.contains(':') && !host.starts_with('[') { format!("[{host}]") } else { - host.clone() + host.to_string() }; let port = port.unwrap_or(HDFS_DEFAULT_PORT); // Validated like every other NameNode spelling, so a host that @@ -227,17 +234,15 @@ fn hdfs_native_config_parse_with( .entry(FS_DEFAULT_FS.to_string()) .or_insert(default_fs); } - None if port.is_some() => { - warn(format!( - "`{HDFS_PORT}` has no effect without `{HDFS_HOST}` and is ignored" - )); - } - None => {} + } + + let mut cfg = HdfsNativeConfig::default(); + if !props.name_node.is_empty() { + cfg.name_node = Some(props.name_node.join(",")); } if !options.is_empty() { cfg.options = Some(options); } - Ok(cfg) } From 4ab34eefc326d49315f08022bdcf75d6b00b7641 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 22:37:25 +0300 Subject: [PATCH 41/51] ci: run the HDFS tests in the default suite like the other fixtures The HDFS fixture joins the shared bridge network with published ports instead of using host networking behind a compose profile, so make docker-up starts it on every platform: the DataNode advertises 127.0.0.1 and the tests set dfs.client.use.datanode.hostname. The tests no longer skip without ICEBERG_TEST_HDFS_ENDPOINT, and the dedicated Tests (hdfs) job is gone. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 42 +----------- .../opendal/tests/file_io_hdfs_test.rs | 65 +++++-------------- dev/docker-compose.yaml | 21 +++--- dev/hdfs/core-site.xml | 2 +- dev/hdfs/hdfs-site.xml | 7 ++ 5 files changed, 35 insertions(+), 102 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baf0d3fe05..2c57b4799c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,46 +265,6 @@ jobs: if: always() && matrix.test-suite.name == 'default' run: make docker-down - tests-hdfs: - needs: lint - runs-on: ubuntu-latest - name: Tests (hdfs) - env: - CARGO_INCREMENTAL: "0" - CARGO_PROFILE_DEV_DEBUG: "0" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Setup Rust toolchain - uses: $/.github/actions/setup-builder - - - name: Cache Rust artifacts - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - key: hdfs - save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - - - name: Install cargo-nextest - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8 - with: - tool: cargo-nextest - - # Only the two HDFS services: the fixture needs host networking and a - # 2 GB image, so it stays out of the default suite's docker-up. - - name: Start HDFS fixture - run: docker compose -f dev/docker-compose.yaml --profile hdfs up -d --wait hdfs-namenode hdfs-datanode - - - name: Run HDFS integration tests - env: - ICEBERG_TEST_HDFS_ENDPOINT: hdfs://localhost:8020 - run: cargo nextest run -p iceberg-storage-opendal --features opendal-hdfs-native --test file_io_hdfs_test - - - name: Stop HDFS fixture - if: always() - run: docker compose -f dev/docker-compose.yaml --profile hdfs down -v --remove-orphans --timeout 0 - msrv: name: Verify MSRV needs: lint @@ -327,7 +287,7 @@ jobs: ci-required: if: ${{ always() }} - needs: [lint, clippy, build, check_standalone, build_with_no_default_features, tests, tests-hdfs, msrv] + needs: [lint, clippy, build, check_standalone, build_with_no_default_features, tests, msrv] runs-on: ubuntu-slim steps: - name: Verify CI jobs succeeded diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index 0871cd46c8..8e410c267a 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -17,16 +17,8 @@ //! Integration tests for HDFS FileIO via OpenDAL `services-hdfs-native`. //! -//! These tests need the HDFS fixture in `dev/docker-compose.yaml` and are -//! skipped when `ICEBERG_TEST_HDFS_ENDPOINT` is not set. The fixture uses -//! host networking (Linux, or a Docker runtime that supports it), so it sits -//! behind a compose profile: -//! -//! ```text -//! COMPOSE_PROFILES=hdfs make docker-up -//! ICEBERG_TEST_HDFS_ENDPOINT=hdfs://localhost:8020 cargo test -p iceberg-storage-opendal \ -//! --features opendal-hdfs-native --test file_io_hdfs_test -//! ``` +//! These tests need the HDFS fixture in `dev/docker-compose.yaml`, which +//! `make docker-up` starts with the other fixtures. #[cfg(feature = "opendal-hdfs-native")] mod tests { @@ -35,29 +27,21 @@ mod tests { use bytes::Bytes; use futures::StreamExt; - use iceberg::io::{FileIO, FileIOBuilder, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; - use iceberg_storage_opendal::{OpenDalResolvingStorageFactory, OpenDalStorageFactory}; - use iceberg_test_utils::{ - ENV_HDFS_ENDPOINT, get_hdfs_endpoint, normalize_test_name_with_parts, set_up, + use iceberg::io::{ + FileIO, FileIOBuilder, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT, StorageFactory, }; + use iceberg_storage_opendal::{OpenDalResolvingStorageFactory, OpenDalStorageFactory}; + use iceberg_test_utils::{get_hdfs_endpoint, normalize_test_name_with_parts, set_up}; - /// Skips the calling test unless the HDFS fixture endpoint is configured; - /// an unset *or* empty variable means "not provided" (see the HF tests). - macro_rules! require_hdfs { - () => { - match std::env::var(ENV_HDFS_ENDPOINT) { - Ok(v) if !v.is_empty() => {} - _ => { - eprintln!("Skipping HDFS test: {} not set", ENV_HDFS_ENDPOINT); - return; - } - } - }; + /// The fixture's DataNode advertises `127.0.0.1`; without this the client + /// dials its container IP, which the host cannot reach on macOS or Windows. + fn file_io_builder(factory: Arc) -> FileIOBuilder { + set_up(); + FileIOBuilder::new(factory).with_prop("hadoop.dfs.client.use.datanode.hostname", "true") } fn get_file_io() -> FileIO { - set_up(); - FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)).build() + file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)).build() } fn test_path(suffix: &str) -> String { @@ -86,7 +70,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_exists() { - require_hdfs!(); let file_io = get_file_io(); let dir = test_path("test_file_io_hdfs_exists"); let present = format!("{dir}/present"); @@ -104,7 +87,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_write_and_read() { - require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_write_and_read"); let _ = file_io.delete(&path).await; @@ -128,10 +110,8 @@ mod tests { /// (the single-node fixture is listed twice to drive the list path). #[tokio::test] async fn test_file_io_hdfs_declared_nameservice() { - require_hdfs!(); - set_up(); let endpoint = get_hdfs_endpoint(); - let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) + let file_io = file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop( format!("{HDFS_NAME_NODE}.logical-nameservice"), format!("{endpoint},{endpoint}"), @@ -163,14 +143,12 @@ mod tests { /// PyIceberg's `hdfs.host`/`hdfs.port`; both must reach the same cluster. #[tokio::test] async fn test_file_io_hdfs_authority_less_paths() { - require_hdfs!(); - set_up(); let endpoint = get_hdfs_endpoint(); let url = url::Url::parse(&endpoint).unwrap(); - let by_name_node = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) + let by_name_node = file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop(HDFS_NAME_NODE, &endpoint) .build(); - let by_host_port = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) + let by_host_port = file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop(HDFS_HOST, url.host_str().unwrap()) .with_prop(HDFS_PORT, url.port().unwrap().to_string()) .build(); @@ -197,7 +175,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_overwrite() { - require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_overwrite"); let _ = file_io.delete(&path).await; @@ -219,7 +196,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete_stream() { - require_hdfs!(); let file_io = get_file_io(); let paths: Vec = (0..5) @@ -250,7 +226,6 @@ mod tests { /// tests; this proves the multi-operator path completes and cleans up. #[tokio::test] async fn test_file_io_hdfs_delete_stream_two_name_nodes() { - require_hdfs!(); let endpoint = get_hdfs_endpoint(); let Some(alternate) = alternate_endpoint(&endpoint) else { eprintln!("Skipping HDFS test: {endpoint} has no second spelling"); @@ -291,7 +266,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete_stream_empty() { - require_hdfs!(); let file_io = get_file_io(); let stream = futures::stream::empty().boxed(); file_io.delete_stream(stream).await.unwrap(); @@ -299,9 +273,7 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_resolving_storage() { - require_hdfs!(); - set_up(); - let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())).build(); + let file_io = file_io_builder(Arc::new(OpenDalResolvingStorageFactory::new())).build(); let path = test_path("test_file_io_hdfs_resolving_storage"); let _ = file_io.delete(&path).await; @@ -322,7 +294,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_metadata() { - require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_metadata"); let _ = file_io.delete(&path).await; @@ -341,7 +312,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete() { - require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_delete"); @@ -359,7 +329,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_delete_prefix() { - require_hdfs!(); let file_io = get_file_io(); let dir = test_path("test_file_io_hdfs_delete_prefix"); let _ = file_io.delete_prefix(&dir).await; @@ -384,7 +353,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_reader_range() { - require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_reader_range"); let _ = file_io.delete(&path).await; @@ -410,7 +378,6 @@ mod tests { #[tokio::test] async fn test_file_io_hdfs_streaming_writer() { - require_hdfs!(); let file_io = get_file_io(); let path = test_path("test_file_io_hdfs_streaming_writer"); let _ = file_io.delete(&path).await; diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 4c0b406847..3bf9d730b5 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -140,18 +140,17 @@ services: # ============================================================================= # HDFS - single-node NameNode + DataNode for HDFS tests # ============================================================================= - # hdfs-native connects to DataNodes by their registered IP โ€” unroutable on - # a docker bridge, hence host networking (needs Linux or a recent runtime). - # Opt-in: `COMPOSE_PROFILES=hdfs make docker-up`. + # The DataNode advertises 127.0.0.1, so clients that set + # `dfs.client.use.datanode.hostname` reach it through its published port. hdfs-namenode: image: apache/hadoop:3.5.0 - profiles: [hdfs] - network_mode: "host" + networks: + iceberg_test: command: ["hdfs", "namenode"] environment: ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" - extra_hosts: - - "docker-desktop:127.0.0.1" + ports: + - "8020:8020" volumes: - ./hdfs/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro - ./hdfs/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro @@ -164,14 +163,14 @@ services: hdfs-datanode: image: apache/hadoop:3.5.0 - profiles: [hdfs] - network_mode: "host" + networks: + iceberg_test: command: ["hdfs", "datanode"] depends_on: hdfs-namenode: condition: service_healthy - extra_hosts: - - "docker-desktop:127.0.0.1" + ports: + - "9866:9866" volumes: - ./hdfs/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro - ./hdfs/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro diff --git a/dev/hdfs/core-site.xml b/dev/hdfs/core-site.xml index 3c57323d7b..273676bba2 100644 --- a/dev/hdfs/core-site.xml +++ b/dev/hdfs/core-site.xml @@ -18,7 +18,7 @@ fs.defaultFS - hdfs://localhost:8020 + hdfs://hdfs-namenode:8020 hadoop.http.staticuser.user diff --git a/dev/hdfs/hdfs-site.xml b/dev/hdfs/hdfs-site.xml index 5eb2d198f1..b9bf0c63c1 100644 --- a/dev/hdfs/hdfs-site.xml +++ b/dev/hdfs/hdfs-site.xml @@ -32,4 +32,11 @@ dfs.datanode.data.dir file:///tmp/hadoop-root/dfs/data + + + dfs.datanode.hostname + 127.0.0.1 + From 0bf5d5aa980d57f9e5c1cb5c0bb51cedea296d40 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 22:47:23 +0300 Subject: [PATCH 42/51] fix(storage): build HDFS clients in a crate-owned runtime An hdfs-native client spawns its NameNode connections on the runtime current when it is built and panics once that runtime is dropped. The cache used to detect that with a sentinel task per entry, which needed Handle::try_current, and the build ran under spawn_blocking, which made create_operator async for every backend. Clients are now built inside a crate-owned runtime started on first use, so a cached operator works from any caller's runtime: the sentinel, Handle::try_current and spawn_blocking are gone, and create_operator is sync again as on main. The Hadoop XML read (~0.1 ms) runs inline. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 301 ++++-------------- crates/storage/opendal/src/lib.rs | 22 +- .../opendal/tests/hdfs_native_runtime_test.rs | 8 +- 3 files changed, 78 insertions(+), 253 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 1e22117590..b93785bc55 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -18,8 +18,7 @@ //! HDFS storage backend via OpenDAL's `services-hdfs-native` (pure Rust, no JNI). use std::collections::HashMap; -use std::collections::hash_map::Entry; -use std::sync::{Arc, PoisonError, RwLock, Weak}; +use std::sync::{Arc, LazyLock, PoisonError, RwLock}; use iceberg::io::{HDFS_HADOOP_CONF_PREFIX, HDFS_HOST, HDFS_NAME_NODE, HDFS_PORT}; use iceberg::{Error, ErrorKind, Result}; @@ -27,8 +26,7 @@ use iceberg_property_macro::Properties; use opendal::Operator; use opendal::services::HdfsNativeConfig; use serde::{Deserialize, Serialize}; -use tokio::runtime::Handle; -use tokio::task::JoinHandle; +use tokio::runtime::Runtime; use url::Url; use crate::OpenDalClientConfig; @@ -418,88 +416,31 @@ impl HdfsNativeStorage { } } -/// Operators cached per effective NameNode: each holds an `hdfs-native` -/// client with live RPC connections, whose tasks run on the tokio runtime -/// current when it was built. An entry is rebuilt once that runtime is -/// gone, as `hdfs-native` panics when -/// it spawns onto a dead one. The cache lives as long as the storage that -/// owns it (clones share it). +/// Operators cached per effective NameNode; the cache lives as long as the +/// storage that owns it (clones share it). #[derive(Clone, Debug, Default)] -pub(crate) struct HdfsNativeOperatorCache(Arc>>); +pub(crate) struct HdfsNativeOperatorCache(Arc>>); -#[derive(Debug)] -struct CachedOperator { - operator: Operator, - sentinel: RuntimeSentinel, -} - -/// A task parked on the building runtime that owns the token, so the token -/// outlives it only while that runtime is alive. Aborted on drop so entries -/// do not leave parked tasks behind. -#[derive(Debug)] -struct RuntimeSentinel { - alive: Weak<()>, - task: JoinHandle<()>, -} - -impl RuntimeSentinel { - fn spawn(handle: &Handle) -> Self { - let token = Arc::new(()); - let alive = Arc::downgrade(&token); - let task = handle.spawn(async move { - let _token = token; - std::future::pending::<()>().await - }); - Self { alive, task } - } -} - -impl Drop for RuntimeSentinel { - fn drop(&mut self) { - self.task.abort(); - } -} - -impl CachedOperator { - fn new(operator: Operator, handle: &Handle) -> Self { - Self { - operator, - sentinel: RuntimeSentinel::spawn(handle), - } - } - - fn runtime_alive(&self) -> bool { - self.sentinel.alive.strong_count() > 0 - } -} - -// A poisoned lock is recovered rather than reported: writes only ever -// replace whole entries, so a panic elsewhere cannot leave the map broken. +// A poisoned lock is recovered rather than reported: writes only insert +// whole entries, so a panic elsewhere cannot leave the map broken. impl HdfsNativeOperatorCache { pub(crate) fn get(&self, name_node: &str) -> Option { self.0 .read() .unwrap_or_else(PoisonError::into_inner) .get(name_node) - .filter(|cached| cached.runtime_alive()) - .map(|cached| cached.operator.clone()) + .cloned() } /// Inserts `op` unless a concurrent caller got there first, returning - /// whichever operator the cache now holds; a stale entry is replaced. - fn insert(&self, name_node: String, op: Operator, handle: &Handle) -> Operator { - let mut operators = self.0.write().unwrap_or_else(PoisonError::into_inner); - match operators.entry(name_node) { - Entry::Occupied(entry) if entry.get().runtime_alive() => entry.get().operator.clone(), - Entry::Occupied(mut entry) => { - entry.insert(CachedOperator::new(op.clone(), handle)); - op - } - Entry::Vacant(entry) => { - entry.insert(CachedOperator::new(op.clone(), handle)); - op - } - } + /// whichever operator the cache now holds. + fn insert(&self, name_node: String, op: Operator) -> Operator { + self.0 + .write() + .unwrap_or_else(PoisonError::into_inner) + .entry(name_node) + .or_insert(op) + .clone() } #[cfg(test)] @@ -508,11 +449,23 @@ impl HdfsNativeOperatorCache { } } +/// The runtime `hdfs-native` clients are built in. A client spawns its +/// NameNode connections on the runtime current when it is built and panics +/// once that runtime is gone, so building it here keeps a cached operator +/// usable from any caller's runtime. Started on first use. +static HDFS_NATIVE_RUNTIME: LazyLock = LazyLock::new(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("iceberg-hdfs-native") + .build() + .expect("failed to start the HDFS client runtime") +}); + /// Creates an operator for the path, reusing the cached one for its /// effective NameNode. -pub(crate) async fn hdfs_native_create_operator<'a>( +pub(crate) fn hdfs_native_create_operator<'a>( path: &'a str, - config: &Arc, + config: &HdfsNativeConfig, operators: &HdfsNativeOperatorCache, ) -> Result<(Operator, &'a str)> { let (name_node, relative_path) = hdfs_native_effective_name_node(config, path)?; @@ -521,27 +474,15 @@ pub(crate) async fn hdfs_native_create_operator<'a>( return Ok((op, relative_path)); } - // Every operator in this crate needs a tokio runtime for its I/O (the - // timeout layer), so say so instead of panicking in `spawn_blocking`. - let handle = Handle::try_current().map_err(|_| { - Error::new( - ErrorKind::FeatureUnsupported, - "HDFS storage requires a tokio runtime", - ) - })?; - - // The build reads the Hadoop XML config synchronously, so it runs on a - // blocking thread and outside the lock. A racing first caller may build - // too; the loser is dropped before opening any connection. - let build_config = Arc::clone(config); - let build_name_node = name_node.clone(); - let op = handle - .spawn_blocking(move || hdfs_native_operator_build(&build_config, &build_name_node)) - .await - .map_err(|e| { - Error::new(ErrorKind::Unexpected, "HDFS operator build task failed").with_source(e) - })??; - Ok((operators.insert(name_node, op, &handle), relative_path)) + // Built outside the lock, so a racing first caller may build too; the + // loser is dropped before opening any connection. The build reads the + // Hadoop XML config synchronously (~0.1 ms locally), once per NameNode + // per storage, which is accepted here rather than offloaded. + let op = { + let _runtime = HDFS_NATIVE_RUNTIME.enter(); + hdfs_native_operator_build(config, &name_node)? + }; + Ok((operators.insert(name_node, op), relative_path)) } /// Returns the `delete_stream` grouping key for a path: the effective @@ -794,8 +735,8 @@ mod tests { /// A bare configured NameNode and the same NameNode as a path authority /// share one cache entry. - #[tokio::test] - async fn test_hdfs_native_create_operator_dedupes_spellings() { + #[test] + fn test_hdfs_native_create_operator_dedupes_spellings() { let config = Arc::new( hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), @@ -806,9 +747,7 @@ mod tests { let operators = HdfsNativeOperatorCache::default(); for path in ["hdfs:///a", "hdfs://nn:8020/b"] { - hdfs_native_create_operator(path, &config, &operators) - .await - .unwrap(); + hdfs_native_create_operator(path, &config, &operators).unwrap(); } assert_eq!(operators.len(), 1); @@ -914,8 +853,8 @@ mod tests { assert_eq!(nn, "hdfs://other:9000"); } - #[tokio::test] - async fn test_hdfs_native_create_operator_per_nameservice() { + #[test] + fn test_hdfs_native_create_operator_per_nameservice() { let config = Arc::new( hdfs_native_config_parse(HashMap::from([ ( @@ -933,9 +872,7 @@ mod tests { let operators = HdfsNativeOperatorCache::default(); for path in ["hdfs://ns-a/x", "hdfs://ns-b/y", "hdfs:///w"] { - hdfs_native_create_operator(path, &config, &operators) - .await - .unwrap(); + hdfs_native_create_operator(path, &config, &operators).unwrap(); } // ns-a, ns-b, and the plain key for the authority-less path; an // undeclared nameservice never reaches an operator. @@ -944,9 +881,7 @@ mod tests { hdfs_native_batch_key(&config, "hdfs://ns-b/y"), "hdfs://b:8020" ); - let err = hdfs_native_create_operator("hdfs://ns-c/z", &config, &operators) - .await - .unwrap_err(); + let err = hdfs_native_create_operator("hdfs://ns-c/z", &config, &operators).unwrap_err(); assert!(err.to_string().contains("`hdfs.name-node.ns-c`"), "{err}"); assert_eq!(operators.len(), 3); } @@ -1208,8 +1143,8 @@ mod tests { ); } - #[tokio::test] - async fn test_hdfs_native_create_operator_plain_key_serves_authority_less_paths() { + #[test] + fn test_hdfs_native_create_operator_plain_key_serves_authority_less_paths() { let config = Arc::new( hdfs_native_config_parse(HashMap::from([( HDFS_NAME_NODE.to_string(), @@ -1220,28 +1155,22 @@ mod tests { let operators = HdfsNativeOperatorCache::default(); // Authority-less paths share the configured list's operator. - let (_, rel) = hdfs_native_create_operator("hdfs:///a/b", &config, &operators) - .await - .unwrap(); + let (_, rel) = hdfs_native_create_operator("hdfs:///a/b", &config, &operators).unwrap(); assert_eq!(rel, "a/b"); assert_eq!(operators.len(), 1); assert!(operators.get("hdfs://nn1:8020,hdfs://nn2:8020").is_some()); // A logical nameservice must be declared; a concrete authority is // another cluster, never the configured one. - let err = hdfs_native_create_operator("hdfs://ns-a/c", &config, &operators) - .await - .unwrap_err(); + let err = hdfs_native_create_operator("hdfs://ns-a/c", &config, &operators).unwrap_err(); assert!(err.to_string().contains("`hdfs.name-node.ns-a`"), "{err}"); - hdfs_native_create_operator("hdfs://other:9000/d", &config, &operators) - .await - .unwrap(); + hdfs_native_create_operator("hdfs://other:9000/d", &config, &operators).unwrap(); assert_eq!(operators.len(), 2); assert!(operators.get("hdfs://other:9000").is_some()); } - #[tokio::test] - async fn test_hdfs_native_create_operator_caches_per_name_node() { + #[test] + fn test_hdfs_native_create_operator_caches_per_name_node() { let config = Arc::new(HdfsNativeConfig::default()); let operators = HdfsNativeOperatorCache::default(); @@ -1250,34 +1179,15 @@ mod tests { "hdfs://nn1:8020/b", "hdfs://nn2:8020/c", ] { - hdfs_native_create_operator(path, &config, &operators) - .await - .unwrap(); + hdfs_native_create_operator(path, &config, &operators).unwrap(); } assert_eq!(operators.len(), 2); } - #[test] - fn test_hdfs_native_create_operator_without_runtime_errors() { - let config = Arc::new(HdfsNativeConfig::default()); - let operators = HdfsNativeOperatorCache::default(); - - let err = futures::executor::block_on(hdfs_native_create_operator( - "hdfs://nn:8020/a", - &config, - &operators, - )) - .unwrap_err(); - - assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); - assert_eq!(operators.len(), 0); - } - #[tokio::test] async fn test_hdfs_native_operator_cache_keeps_first_insert() { let operators = HdfsNativeOperatorCache::default(); - let handle = Handle::current(); let build = |root: &str| { let mut config = hdfs_native_config_parse(HashMap::new()).unwrap(); config.root = Some(root.to_string()); @@ -1286,18 +1196,18 @@ mod tests { // Two callers racing for one NameNode: the first insert wins and both // get the cached operator. - let first = operators.insert("hdfs://nn:8020".to_string(), build("/first"), &handle); - let second = operators.insert("hdfs://nn:8020".to_string(), build("/second"), &handle); + let first = operators.insert("hdfs://nn:8020".to_string(), build("/first")); + let second = operators.insert("hdfs://nn:8020".to_string(), build("/second")); assert_eq!(first.info().root(), "/first/"); assert_eq!(second.info().root(), "/first/"); assert_eq!(operators.len(), 1); } - #[tokio::test] - async fn test_hdfs_native_operator_cache_survives_a_poisoned_lock() { + #[test] + fn test_hdfs_native_operator_cache_survives_a_poisoned_lock() { let operators = HdfsNativeOperatorCache::default(); // A panic elsewhere while the lock is held must not break HDFS I/O - // for good: every write replaces a whole entry, so the map is intact. + // for good: every write inserts a whole entry, so the map is intact. let _ = std::panic::catch_unwind(|| { let _guard = operators.0.write().unwrap(); panic!("poisoning the operator cache lock"); @@ -1305,97 +1215,14 @@ mod tests { assert!(operators.0.is_poisoned()); let config = Arc::new(HdfsNativeConfig::default()); - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) - .await - .unwrap(); + hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators).unwrap(); assert!(operators.get("hdfs://nn:8020").is_some()); } - #[test] - fn test_hdfs_native_operator_cache_rebuilds_after_runtime_shutdown() { - let flavors: [fn() -> tokio::runtime::Runtime; 2] = [ - || { - tokio::runtime::Builder::new_current_thread() - .build() - .unwrap() - }, - || tokio::runtime::Builder::new_multi_thread().build().unwrap(), - ]; - for build_runtime in flavors { - let operators = HdfsNativeOperatorCache::default(); - let config = Arc::new(HdfsNativeConfig::default()); - - let runtime = build_runtime(); - let (stale, _) = runtime - .block_on(hdfs_native_create_operator( - "hdfs://nn:8020/a", - &config, - &operators, - )) - .unwrap(); - assert!(operators.get("hdfs://nn:8020").is_some()); - - // The building runtime is gone: the entry is stale and the next - // runtime to use it builds a new operator in its place. - drop(runtime); - assert!(operators.get("hdfs://nn:8020").is_none()); - let next = build_runtime(); - let (rebuilt, _) = next - .block_on(hdfs_native_create_operator( - "hdfs://nn:8020/a", - &config, - &operators, - )) - .unwrap(); - assert!(operators.get("hdfs://nn:8020").is_some()); - assert_eq!(operators.len(), 1); - let (_, stale) = stale.into_parts(); - let (_, rebuilt) = rebuilt.into_parts(); - assert!( - !Arc::ptr_eq(&stale, &rebuilt), - "the stale operator was reused" - ); - } - } - - #[test] - fn test_hdfs_native_operator_cache_drops_its_sentinel_task() { - let flavors: [fn() -> tokio::runtime::Runtime; 2] = [ - || { - tokio::runtime::Builder::new_current_thread() - .build() - .unwrap() - }, - || tokio::runtime::Builder::new_multi_thread().build().unwrap(), - ]; - for build_runtime in flavors { - build_runtime().block_on(async { - let metrics = Handle::current().metrics(); - let baseline = metrics.num_alive_tasks(); - - let operators = HdfsNativeOperatorCache::default(); - let config = Arc::new(HdfsNativeConfig::default()); - hdfs_native_create_operator("hdfs://nn:8020/a", &config, &operators) - .await - .unwrap(); - assert!(metrics.num_alive_tasks() > baseline, "no sentinel task"); - - // Dropping the cache aborts the sentinel; the abort lands once - // the runtime schedules the task. - drop(operators); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - while metrics.num_alive_tasks() != baseline { - assert!(std::time::Instant::now() < deadline, "sentinel task leaked"); - tokio::task::yield_now().await; - } - }); - } - } - /// The configuration round-trips through serde; the operator cache does /// not and starts empty. - #[tokio::test] - async fn test_hdfs_native_storage_serde_round_trip() { + #[test] + fn test_hdfs_native_storage_serde_round_trip() { let props = HashMap::from([ (HDFS_NAME_NODE.to_string(), "hdfs://nn:8020".to_string()), ( @@ -1416,9 +1243,7 @@ mod tests { let crate::OpenDalStorage::HdfsNative(hdfs) = &storage else { panic!("expected the HdfsNative variant"); }; - hdfs_native_create_operator("hdfs:///x", &hdfs.config, &hdfs.operators) - .await - .unwrap(); + hdfs_native_create_operator("hdfs:///x", &hdfs.config, &hdfs.operators).unwrap(); assert!(hdfs.operators.get("hdfs://nn:8020").is_some()); let value = serde_json::to_value(&storage).unwrap(); diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 660e64ba7d..693f4c18b3 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -394,7 +394,7 @@ impl OpenDalStorage { /// * An [`opendal::Operator`] instance used to operate on file. /// * Relative path to the root uri of [`opendal::Operator`]. #[allow(unreachable_code, unused_variables)] - pub(crate) async fn create_operator<'a>( + pub(crate) fn create_operator<'a>( &self, path: &'a impl AsRef, ) -> Result<(Operator, &'a str)> { @@ -459,7 +459,7 @@ impl OpenDalStorage { } #[cfg(feature = "opendal-hdfs-native")] OpenDalStorage::HdfsNative(storage) => { - hdfs_native_create_operator(path, &storage.config, &storage.operators).await? + hdfs_native_create_operator(path, &storage.config, &storage.operators)? } #[cfg(feature = "opendal-oss")] OpenDalStorage::Oss { config, .. } => { @@ -672,12 +672,12 @@ impl OpenDalStorage { #[async_trait] impl Storage for OpenDalStorage { async fn exists(&self, path: &str) -> Result { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; Ok(op.exists(relative_path).await.map_err(from_opendal_error)?) } async fn metadata(&self, path: &str) -> Result { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; let meta = op.stat(relative_path).await.map_err(from_opendal_error)?; Ok(FileMetadata { size: meta.content_length(), @@ -685,7 +685,7 @@ impl Storage for OpenDalStorage { } async fn read(&self, path: &str) -> Result { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; Ok(op .read(relative_path) .await @@ -694,14 +694,14 @@ impl Storage for OpenDalStorage { } async fn reader(&self, path: &str) -> Result> { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; Ok(Box::new(OpenDalReader( op.reader(relative_path).await.map_err(from_opendal_error)?, ))) } async fn write(&self, path: &str, bs: Bytes) -> Result<()> { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; op.write(relative_path, bs) .await .map_err(from_opendal_error)?; @@ -709,19 +709,19 @@ impl Storage for OpenDalStorage { } async fn writer(&self, path: &str) -> Result> { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; Ok(Box::new(OpenDalWriter::new( op.writer(relative_path).await.map_err(from_opendal_error)?, ))) } async fn delete(&self, path: &str) -> Result<()> { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; Ok(op.delete(relative_path).await.map_err(from_opendal_error)?) } async fn delete_prefix(&self, path: &str) -> Result<()> { - let (op, relative_path) = self.create_operator(&path).await?; + let (op, relative_path) = self.create_operator(&path)?; let path = if relative_path.ends_with('/') { relative_path.to_string() } else { @@ -745,7 +745,7 @@ impl Storage for OpenDalStorage { (self.relativize_path(&path)?.to_string(), entry.into_mut()) } Entry::Vacant(entry) => { - let (op, rel) = self.create_operator(&path).await?; + let (op, rel) = self.create_operator(&path)?; let rel = rel.to_string(); let deleter = op.deleter().await.map_err(from_opendal_error)?; (rel, entry.insert(deleter)) diff --git a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs index 2235314e4d..da22260d72 100644 --- a/crates/storage/opendal/tests/hdfs_native_runtime_test.rs +++ b/crates/storage/opendal/tests/hdfs_native_runtime_test.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -//! A cached HDFS operator is bound to the tokio runtime that built it; once -//! that runtime is gone it must be rebuilt rather than reused. Needs no -//! HDFS: the NameNode is a local listener that only counts dials. +//! A cached HDFS operator must keep working after the runtime that first +//! used it is dropped. Needs no HDFS: the NameNode is a local listener that +//! only counts dials. #[cfg(feature = "opendal-hdfs-native")] mod tests { @@ -55,7 +55,7 @@ mod tests { } #[test] - fn test_hdfs_operator_is_rebuilt_after_its_runtime_is_dropped() { + fn test_hdfs_operator_outlives_the_runtime_that_first_used_it() { let (port, dials) = fake_name_node(); let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop(HDFS_NAME_NODE, format!("hdfs://127.0.0.1:{port}")) From f092ea2f035d3936d753011827d4515b520371ac Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 22:58:31 +0300 Subject: [PATCH 43/51] refactor(storage): type HDFS path authorities and validate declarations at parse hdfs_native_parse_path returns an HdfsNativeAuthority (NameNode, Nameservice, Default), so the resolver matches on it instead of re-parsing the authority to learn whether it has a port. Every HA nameservice declaration, from hdfs.name-node. or passed through as hadoop.dfs.ha.namenodes.*, and the fs.defaultFS that serves authority-less paths are now validated when the config is parsed, so a broken one fails the first call rather than only the paths that use it. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 181 +++++++++++++++------- 1 file changed, 122 insertions(+), 59 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index b93785bc55..67456e5b21 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -241,6 +241,18 @@ fn hdfs_native_config_parse_with( if !options.is_empty() { cfg.options = Some(options); } + + // Every declaration, sugar or passed through, and the default filesystem + // fail here rather than at the first path that uses them. + for nameservice in cfg.options.iter().flat_map(|options| { + options.keys().filter_map(|key| { + key.strip_prefix(HA_NAMENODES_PREFIX) + .and_then(|rest| rest.strip_prefix('.')) + }) + }) { + hdfs_native_nameservice(&cfg, nameservice)?; + } + hdfs_native_default_name_node(&cfg)?; Ok(cfg) } @@ -271,9 +283,20 @@ fn hdfs_native_invalid_path(path: &str, reason: impl std::fmt::Display) -> Error ) } -/// Parse an HDFS path into `Some("hdfs://")` (`None` when -/// authority-less) and the relative path (no leading `/`, opendal style). -pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str)> { +/// A path's authority, as the resolver reads it. +#[derive(Debug, PartialEq)] +pub(crate) enum HdfsNativeAuthority { + /// `hdfs://host:port`, normalized: dialed as is. + NameNode(String), + /// `hdfs://name` without a port: a nameservice that must be declared. + Nameservice(String), + /// `hdfs:///path`: served by `hdfs.name-node`, else `fs.defaultFS`. + Default, +} + +/// Parse an HDFS path into its authority and the relative path (no leading +/// `/`, opendal style). +pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(HdfsNativeAuthority, &str)> { let url = Url::parse(path).map_err(|e| hdfs_native_invalid_path(path, e))?; // Non-special schemes parse even without `//` (e.g. `hdfs:x` is a valid // non-hierarchical URL), so require the literal prefix before slicing. @@ -289,11 +312,11 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str return Err(hdfs_native_invalid_path(path, "port 0 cannot be dialed")); } - let name_node = url.host_str().filter(|h| !h.is_empty()).map(|host| { - url.port() - .map(|port| format!("hdfs://{host}:{port}")) - .unwrap_or_else(|| format!("hdfs://{host}")) - }); + let authority = match (url.host_str().filter(|h| !h.is_empty()), url.port()) { + (Some(host), Some(port)) => HdfsNativeAuthority::NameNode(format!("hdfs://{host}:{port}")), + (Some(host), None) => HdfsNativeAuthority::Nameservice(host.to_string()), + (None, _) => HdfsNativeAuthority::Default, + }; // `url.path()` borrows from `url` and can't be returned with the input's // lifetime. Slice the path component out of the original input instead; @@ -304,7 +327,7 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(Option, &str None => "", }; - Ok((name_node, rel)) + Ok((authority, rel)) } /// Resolves the effective NameNode for a path, plus the relative path. An @@ -317,43 +340,52 @@ pub(crate) fn hdfs_native_effective_name_node<'a>( path: &'a str, ) -> Result<(String, &'a str)> { let (authority, relative_path) = hdfs_native_parse_path(path)?; - let invalid = |reason: String| hdfs_native_invalid_path(path, reason); let name_node = match authority { - Some(authority) if hdfs_native_name_node(&authority).is_some() => authority, - Some(portless) => { - let nameservice = portless.trim_start_matches("hdfs://"); - hdfs_native_nameservice(config, nameservice)?.ok_or_else(|| { - invalid(format!( - "`{nameservice}` has no port and is not a declared nameservice; add the port or set `{HDFS_NAME_NODE}.{nameservice}`" - )) + HdfsNativeAuthority::NameNode(name_node) => name_node, + HdfsNativeAuthority::Nameservice(nameservice) => { + hdfs_native_nameservice(config, &nameservice)?.ok_or_else(|| { + hdfs_native_invalid_path( + path, + format!( + "`{nameservice}` has no port and is not a declared nameservice; add the port or set `{HDFS_NAME_NODE}.{nameservice}`" + ), + ) })? } - None => match (&config.name_node, hdfs_native_default_fs(config)) { - (Some(name_node), _) => name_node.clone(), - (None, Some(default_fs)) => hdfs_native_name_node(default_fs).ok_or_else(|| { - invalid(format!( - "`{FS_DEFAULT_FS}` {} is not an HDFS host:port, a logical nameservice requires `{HDFS_NAME_NODE}`", - hdfs_native_redact(default_fs) - )) - })?, - (None, None) => { - return Err(invalid(format!( - "authority-less paths require `{HDFS_NAME_NODE}` or `{HDFS_HOST}`" - ))); - } - }, + HdfsNativeAuthority::Default => hdfs_native_default_name_node(config)?.ok_or_else(|| { + hdfs_native_invalid_path( + path, + format!("authority-less paths require `{HDFS_NAME_NODE}` or `{HDFS_HOST}`"), + ) + })?, }; Ok((name_node, relative_path)) } -/// `fs.defaultFS` from the forwarded options, as written. -fn hdfs_native_default_fs(config: &HdfsNativeConfig) -> Option<&str> { - config +/// The NameNode that serves authority-less paths: `hdfs.name-node`, else +/// `fs.defaultFS`, which must then be an HDFS `host:port`. +fn hdfs_native_default_name_node(config: &HdfsNativeConfig) -> Result> { + if let Some(name_node) = &config.name_node { + return Ok(Some(name_node.clone())); + } + let Some(default_fs) = config .options - .as_ref()? - .get(FS_DEFAULT_FS) + .as_ref() + .and_then(|options| options.get(FS_DEFAULT_FS)) .map(|s| s.trim()) .filter(|s| !s.is_empty()) + else { + return Ok(None); + }; + hdfs_native_name_node(default_fs).map(Some).ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid `{FS_DEFAULT_FS}`: {} is not an HDFS host:port, a logical nameservice requires `{HDFS_NAME_NODE}`", + hdfs_native_redact(default_fs) + ), + ) + }) } /// The NameNodes that Hadoop's keys in the forwarded options declare for a @@ -705,31 +737,43 @@ mod tests { #[test] fn test_hdfs_native_default_fs_must_be_hdfs() { - let parse = |default_fs: &str| { - hdfs_native_config_parse(HashMap::from([( - "hadoop.fs.defaultFS".to_string(), - default_fs.to_string(), - )])) - .unwrap() + let parse = |props: &[(&str, &str)]| { + hdfs_native_config_parse( + props + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ) }; + let default_fs = |value| parse(&[("hadoop.fs.defaultFS", value)]); let (nn, _) = - hdfs_native_effective_name_node(&parse("hdfs://nn:8020/"), "hdfs:///a").unwrap(); + hdfs_native_effective_name_node(&default_fs("hdfs://nn:8020/").unwrap(), "hdfs:///a") + .unwrap(); assert_eq!(nn, "hdfs://nn:8020"); // Not an HDFS `host:port`, including the portless logical-name form, - // which only a declaration resolves. - for default_fs in ["viewfs://cluster/", "hdfs://", "hdfs://ns1"] { - let err = hdfs_native_effective_name_node(&parse(default_fs), "hdfs:///a").unwrap_err(); + // which only a declaration resolves: rejected when the config is + // parsed, since it serves every authority-less path. + for value in ["viewfs://cluster/", "hdfs://", "hdfs://ns1"] { + let err = default_fs(value).unwrap_err(); assert!( err.to_string().contains(FS_DEFAULT_FS) && err.to_string().contains(HDFS_NAME_NODE), - "{default_fs}: {err}" + "{value}: {err}" ); } + // Not checked when `hdfs.name-node` serves those paths instead. + parse(&[ + ("hadoop.fs.defaultFS", "viewfs://cluster/"), + (HDFS_NAME_NODE, "nn:8020"), + ]) + .unwrap(); // Empty is unset, so the authority-less path has nothing to resolve to. - let err = hdfs_native_effective_name_node(&parse(""), "hdfs:///a").unwrap_err(); + let err = + hdfs_native_effective_name_node(&default_fs("").unwrap(), "hdfs:///a").unwrap_err(); assert!(err.to_string().contains(HDFS_HOST), "{err}"); // Normalized like every other source. - let (nn, _) = hdfs_native_effective_name_node(&parse("nn:9000"), "hdfs:///a").unwrap(); + let (nn, _) = + hdfs_native_effective_name_node(&default_fs("nn:9000").unwrap(), "hdfs:///a").unwrap(); assert_eq!(nn, "hdfs://nn:9000"); } @@ -807,13 +851,17 @@ mod tests { .unwrap_err(); assert!(err.to_string().contains(&key), "{key}={value}: {err}"); } - // A declaration with a broken address is an error, not a silent fallback. - let broken = hdfs_native_config_parse(HashMap::from([( + // A passed-through declaration with a missing address fails when the + // config is parsed, not at the first path that names it. + let err = hdfs_native_config_parse(HashMap::from([( "hadoop.dfs.ha.namenodes.ns-d".to_string(), "a".to_string(), )])) - .unwrap(); - assert!(hdfs_native_nameservice(&broken, "ns-d").is_err()); + .unwrap_err(); + assert!( + err.to_string().contains("dfs.namenode.rpc-address.ns-d.a"), + "{err}" + ); } #[test] @@ -929,7 +977,10 @@ mod tests { fn test_hdfs_native_parse_path_with_authority_and_rel() { let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1/a/b").unwrap(); - assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); + assert_eq!( + nn, + HdfsNativeAuthority::Nameservice("nameservice1".to_string()) + ); assert_eq!(rel, "a/b"); } @@ -937,14 +988,20 @@ mod tests { fn test_hdfs_native_parse_path_with_authority_and_port() { let (nn, rel) = hdfs_native_parse_path("hdfs://nn:8020/foo").unwrap(); - assert_eq!(nn.as_deref(), Some("hdfs://nn:8020")); + assert_eq!( + nn, + HdfsNativeAuthority::NameNode("hdfs://nn:8020".to_string()) + ); assert_eq!(rel, "foo"); } #[test] fn test_hdfs_native_parse_path_ipv6_authority_keeps_brackets() { let (nn, rel) = hdfs_native_parse_path("hdfs://[::1]:8020/a/b").unwrap(); - assert_eq!(nn.as_deref(), Some("hdfs://[::1]:8020")); + assert_eq!( + nn, + HdfsNativeAuthority::NameNode("hdfs://[::1]:8020".to_string()) + ); assert_eq!(rel, "a/b"); } @@ -1027,7 +1084,10 @@ mod tests { fn test_hdfs_native_parse_path_with_authority_no_path() { let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1").unwrap(); - assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); + assert_eq!( + nn, + HdfsNativeAuthority::Nameservice("nameservice1".to_string()) + ); assert_eq!(rel, ""); } @@ -1035,7 +1095,10 @@ mod tests { fn test_hdfs_native_parse_path_with_authority_trailing_slash() { let (nn, rel) = hdfs_native_parse_path("hdfs://nameservice1/").unwrap(); - assert_eq!(nn.as_deref(), Some("hdfs://nameservice1")); + assert_eq!( + nn, + HdfsNativeAuthority::Nameservice("nameservice1".to_string()) + ); assert_eq!(rel, ""); } @@ -1043,7 +1106,7 @@ mod tests { fn test_hdfs_native_parse_path_authority_less_returns_none() { let (nn, rel) = hdfs_native_parse_path("hdfs:///a/b").unwrap(); - assert_eq!(nn, None); + assert_eq!(nn, HdfsNativeAuthority::Default); assert_eq!(rel, "a/b"); } From 26aba0332ebd01cf203d47970389109d70159cdd Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 23:05:47 +0300 Subject: [PATCH 44/51] refactor(storage): drop index slicing and combinator chains in the HDFS backend The nameservice lookup is a for loop with let-else, and the userinfo masking and the relative-path extraction use split_once/rsplit_once instead of slicing at computed indices. A test pins that doubled slashes after the authority are still trimmed. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 74 ++++++++++++----------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 67456e5b21..e9e1515faf 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -261,16 +261,19 @@ fn hdfs_native_config_parse_with( /// `?`, `#` or `://` that a URL parser would split elsewhere. An `@` further /// along the path is masked the same way, which only costs context. fn hdfs_native_redact(s: &str) -> String { - let start = s - .find("://") - .filter(|&i| { - s[..i] + // Only a real scheme is kept; anything else before `://` may be userinfo. + let (scheme, rest) = match s.split_once("://") { + Some((scheme, rest)) + if scheme .chars() - .all(|c| c.is_ascii_alphanumeric() || "+-.".contains(c)) - }) - .map_or(0, |i| i + 3); - match s[start..].rfind('@') { - Some(at) => format!("{}***{}", &s[..start], &s[start + at..]), + .all(|c| c.is_ascii_alphanumeric() || "+-.".contains(c)) => + { + (format!("{scheme}://"), rest) + } + _ => (String::new(), s), + }; + match rest.rsplit_once('@') { + Some((_, after)) => format!("{scheme}***@{after}"), None => s.to_string(), } } @@ -319,13 +322,12 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(HdfsNativeAuthority, }; // `url.path()` borrows from `url` and can't be returned with the input's - // lifetime. Slice the path component out of the original input instead; - // it starts after the first `/` following the `hdfs://` prefix. Opendal - // paths must not start with `/` (`Deleter::delete` rejects them). - let rel = match after_scheme.find('/') { - Some(i) => after_scheme[i..].trim_start_matches('/'), - None => "", - }; + // lifetime. Take the path from the original input instead: everything + // after the first `/` following the `hdfs://` prefix. Opendal paths must + // not start with `/` (`Deleter::delete` rejects them). + let rel = after_scheme + .split_once('/') + .map_or("", |(_, path)| path.trim_start_matches('/')); Ok((authority, rel)) } @@ -398,25 +400,22 @@ fn hdfs_native_nameservice(config: &HdfsNativeConfig, nameservice: &str) -> Resu let Some(ids) = options.get(&format!("{HA_NAMENODES_PREFIX}.{nameservice}")) else { return Ok(None); }; - let name_nodes = ids - .split(',') - .map(str::trim) - .filter(|id| !id.is_empty()) - .map(|id| { - let key = format!("{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}"); - options - .get(&key) - .and_then(|value| hdfs_native_name_node(value)) - .ok_or_else(|| { - Error::new( - ErrorKind::DataInvalid, - format!( - "Nameservice `{nameservice}` declares NameNode `{id}` but `{key}` is missing or not host:port" - ), - ) - }) - }) - .collect::>>()?; + let mut name_nodes = Vec::new(); + for id in ids.split(',').map(str::trim).filter(|id| !id.is_empty()) { + let key = format!("{HA_NAMENODE_RPC_ADDRESS_PREFIX}.{nameservice}.{id}"); + let Some(name_node) = options + .get(&key) + .and_then(|value| hdfs_native_name_node(value)) + else { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Nameservice `{nameservice}` declares NameNode `{id}` but `{key}` is missing or not host:port" + ), + )); + }; + name_nodes.push(name_node); + } if name_nodes.is_empty() { return Err(Error::new( ErrorKind::DataInvalid, @@ -987,6 +986,11 @@ mod tests { #[test] fn test_hdfs_native_parse_path_with_authority_and_port() { let (nn, rel) = hdfs_native_parse_path("hdfs://nn:8020/foo").unwrap(); + // Extra leading slashes go too: opendal rejects paths starting with `/`. + assert_eq!( + hdfs_native_parse_path("hdfs://nn:8020//foo").unwrap().1, + "foo" + ); assert_eq!( nn, From 80f2149d7b0b89f5d24d2be1d59592d099ae4a83 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 23:12:48 +0300 Subject: [PATCH 45/51] fix(storage): attach the hdfs.port parse error as the source The parse error was interpolated into the message; it is now attached as the error's source, which iceberg prints after the message. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index e9e1515faf..9a1e094a5c 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -194,8 +194,9 @@ fn hdfs_native_config_parse_with( port => Some(port.parse::().map_err(|e| { Error::new( ErrorKind::DataInvalid, - format!("Invalid `{HDFS_PORT}`: {port}: {e}"), + format!("Invalid `{HDFS_PORT}`: {port}"), ) + .with_source(e) })?), }; // PyIceberg's `hdfs.host`/`hdfs.port` name the filesystem for @@ -655,6 +656,11 @@ mod tests { ] { let err = parse(props).unwrap_err(); assert!(err.to_string().contains(HDFS_PORT), "{props:?}: {err}"); + // The parse error rides along as the source, not in the message. + assert!( + std::error::Error::source(&err).is_some(), + "{props:?}: {err}" + ); } // The composed value is validated like any other NameNode spelling. for props in [ From 323c94bcd96f012093ff2832ec8570a790624937 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 23:20:33 +0300 Subject: [PATCH 46/51] fix(storage): share one HDFS client across host name spellings host_str keeps the host's case for hdfs:// URLs, so hdfs://NN1:8020 and hdfs://nn1:8020 cached two clients for one NameNode. Endpoint hosts are now lowercased by one helper used by the config normalizer and the path parser, as Hadoop's FileSystem cache key does; nameservice names stay case-sensitive like the Hadoop keys that declare them. The cache's doc states that it is unbounded by design. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index 9a1e094a5c..ac7ce776d4 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -61,7 +61,14 @@ fn hdfs_native_name_node(entry: &str) -> Option { || !matches!(url.path(), "" | "/") || url.query().is_some() || url.fragment().is_some(); - (!plain).then(|| format!("hdfs://{host}:{port}")) + (!plain).then(|| hdfs_native_endpoint(host, port)) +} + +/// An endpoint as every source spells it, the host lowercased because host +/// names are case-insensitive: `NN1:8020` and `nn1:8020` share one client, +/// as they share one `FileSystem` in Hadoop's cache. +fn hdfs_native_endpoint(host: &str, port: u16) -> String { + format!("hdfs://{}:{port}", host.to_ascii_lowercase()) } /// Parses a comma-separated NameNode list, each entry normalized; an empty @@ -317,7 +324,7 @@ pub(crate) fn hdfs_native_parse_path(path: &str) -> Result<(HdfsNativeAuthority, } let authority = match (url.host_str().filter(|h| !h.is_empty()), url.port()) { - (Some(host), Some(port)) => HdfsNativeAuthority::NameNode(format!("hdfs://{host}:{port}")), + (Some(host), Some(port)) => HdfsNativeAuthority::NameNode(hdfs_native_endpoint(host, port)), (Some(host), None) => HdfsNativeAuthority::Nameservice(host.to_string()), (None, _) => HdfsNativeAuthority::Default, }; @@ -448,8 +455,10 @@ impl HdfsNativeStorage { } } -/// Operators cached per effective NameNode; the cache lives as long as the -/// storage that owns it (clones share it). +/// Operators cached per effective NameNode: one per endpoint or nameservice +/// the storage's paths reach, which in practice is a handful of clusters, so +/// it is not bounded. The cache lives as long as the storage that owns it +/// (clones share it). #[derive(Clone, Debug, Default)] pub(crate) struct HdfsNativeOperatorCache(Arc>>); @@ -598,6 +607,7 @@ mod tests { (" hdfs://nn:8020/ ", "hdfs://nn:8020"), ("nn:8020", "hdfs://nn:8020"), ("[::1]:8020", "hdfs://[::1]:8020"), + ("NN:8020", "hdfs://nn:8020"), ( "hdfs://nn1:8020/, nn2:8020,", "hdfs://nn1:8020,hdfs://nn2:8020", @@ -795,7 +805,7 @@ mod tests { ); let operators = HdfsNativeOperatorCache::default(); - for path in ["hdfs:///a", "hdfs://nn:8020/b"] { + for path in ["hdfs:///a", "hdfs://nn:8020/b", "hdfs://NN:8020/c"] { hdfs_native_create_operator(path, &config, &operators).unwrap(); } From 63dbed2ec39ae2368badb8972f9a8d6fcc36b737 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 23:25:50 +0300 Subject: [PATCH 47/51] docs(storage): note the upstream gap behind the HDFS client runtime opendal's hdfs-native service does not expose hdfs-native's ClientBuilder::with_io_runtime (Kimahriman/hdfs-native#255); a TODO at the crate-owned runtime records that a runtime could be passed in once it does. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/src/hdfs_native.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/storage/opendal/src/hdfs_native.rs b/crates/storage/opendal/src/hdfs_native.rs index ac7ce776d4..7e7b4f9cc9 100644 --- a/crates/storage/opendal/src/hdfs_native.rs +++ b/crates/storage/opendal/src/hdfs_native.rs @@ -494,6 +494,9 @@ impl HdfsNativeOperatorCache { /// NameNode connections on the runtime current when it is built and panics /// once that runtime is gone, so building it here keeps a cached operator /// usable from any caller's runtime. Started on first use. +// TODO: pass a runtime to the client instead of entering this one once +// opendal's hdfs-native service exposes hdfs-native's +// `ClientBuilder::with_io_runtime` (Kimahriman/hdfs-native#255). static HDFS_NATIVE_RUNTIME: LazyLock = LazyLock::new(|| { tokio::runtime::Builder::new_multi_thread() .enable_all() From 7ac1f45060b29b2af26a155744f5077213545ca1 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Fri, 9 Oct 2026 23:33:16 +0300 Subject: [PATCH 48/51] test(storage): make the HDFS HA test fail over for real The declared-nameservice test listed the single-node fixture twice, which drove opendal's list path but never needed a failover. Its first NameNode is now a local port nothing listens on, so every call has to fail over through opendal's synthetic nameservice; an opendal change that kept only the first entry of the list would fail it. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/tests/file_io_hdfs_test.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/storage/opendal/tests/file_io_hdfs_test.rs b/crates/storage/opendal/tests/file_io_hdfs_test.rs index 8e410c267a..ef9da20a13 100644 --- a/crates/storage/opendal/tests/file_io_hdfs_test.rs +++ b/crates/storage/opendal/tests/file_io_hdfs_test.rs @@ -106,15 +106,21 @@ mod tests { } /// The HA flow: table locations carry a logical authority, and - /// `hdfs.name-node.` declares its comma-separated endpoints - /// (the single-node fixture is listed twice to drive the list path). + /// `hdfs.name-node.` declares its comma-separated endpoints. + /// The first one refuses connections, so every call must fail over to + /// the fixture through opendal's synthetic nameservice. #[tokio::test] async fn test_file_io_hdfs_declared_nameservice() { let endpoint = get_hdfs_endpoint(); + // A local port nothing listens on once the listener is dropped. + let dead = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); let file_io = file_io_builder(Arc::new(OpenDalStorageFactory::HdfsNative)) .with_prop( format!("{HDFS_NAME_NODE}.logical-nameservice"), - format!("{endpoint},{endpoint}"), + format!("hdfs://{dead},{endpoint}"), ) .build(); From 8aba12ba0454f1c65f0d4eb1bde747a5532019cd Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 10 Oct 2026 00:07:57 +0300 Subject: [PATCH 49/51] chore: retrigger CI Docker Hub rate-limited the previous run's image pulls, so the default test job never started its fixtures. Co-Authored-By: Claude Opus 5.5 From 650e793c94a820f8862fd6083ca6bdaf587da175 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 10 Oct 2026 12:48:19 +0300 Subject: [PATCH 50/51] chore: retrigger CI The previous run hit Docker Hub's unauthenticated pull rate limit again before any test ran. Co-Authored-By: Claude Opus 5.5 From 0145ca6b6bce85a0cd52dda2636f0402e55969f8 Mon Sep 17 00:00:00 2001 From: Michael Taranov Date: Sat, 10 Oct 2026 21:51:39 +0300 Subject: [PATCH 51/51] docs(storage): document how to enable and configure the HDFS backend The iceberg-storage-opendal README gains an HDFS section: enabling the opendal-hdfs-native feature, the factory, the hdfs.* and hadoop.* properties, the Hadoop config, user and Kerberos it reads, and the portless-authority limitation. The experimental-backends note now points to it. Co-Authored-By: Claude Opus 5.5 --- crates/storage/opendal/README.md | 39 +++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/storage/opendal/README.md b/crates/storage/opendal/README.md index ac3a8b62ef..a254433f92 100644 --- a/crates/storage/opendal/README.md +++ b/crates/storage/opendal/README.md @@ -38,7 +38,7 @@ You can enable all stable storage backends at once using the `opendal-all` featu > Note that `opendal-oss` and `opendal-azdls` are currently experimental and not included in `opendal-all`. -> `opendal-hdfs-native` is experimental and not included in `opendal-all`. A portless path authority such as `hdfs://nameservice1/warehouse` must be declared in properties, e.g. `hdfs.name-node.nameservice1 = nn1:8020,nn2:8020`: unlike Hadoop, nameservices defined only in `$HADOOP_CONF_DIR` are not resolved, and a portless host is not dialed on the default port 8020. +> `opendal-hdfs-native` is experimental and not included in `opendal-all`; see [HDFS](#hdfs). ## Usage @@ -85,3 +85,40 @@ async fn main() -> iceberg::Result<()> { Ok(()) } ``` + +## HDFS + +The `opendal-hdfs-native` feature adds an HDFS backend built on +[`hdfs-native`](https://github.com/Kimahriman/hdfs-native), a native Rust client that needs no +JVM. Enable it on its own, since it is not part of `opendal-all`: + +```toml +iceberg-storage-opendal = { version = "x.y.z", features = ["opendal-hdfs-native"] } +``` + +Then pass `OpenDalStorageFactory::HdfsNative` to your catalog builder, or use +`OpenDalResolvingStorageFactory`, which routes `hdfs://` paths to it. The backend reads these +properties: + +| Property | Description | +| --- | --- | +| `hdfs.name-node` | NameNode `host:port` for authority-less paths (`hdfs:///warehouse/...`); a comma-separated list enables HA failover | +| `hdfs.name-node.` | The NameNodes of a nameservice used as a path authority (`hdfs://nameservice1/warehouse/...`), comma-separated | +| `hdfs.host`, `hdfs.port` | PyIceberg's keys for authority-less paths; the port defaults to `8020` | +| `hadoop.*` | Forwarded to the HDFS client with the prefix stripped, e.g. `hadoop.dfs.client.failover.max.attempts` | + +A path with a `host:port` authority is always used as is. For a table under +`hdfs://nameservice1/warehouse` on an HA cluster: + +```text +hdfs.name-node.nameservice1 = nn1.example.com:8020,nn2.example.com:8020 +``` + +The client also loads `core-site.xml` and `hdfs-site.xml` from `$HADOOP_CONF_DIR`, else +`$HADOOP_HOME/etc/hadoop`, and takes the user from `HADOOP_USER_NAME`. Kerberos is used when +`hadoop.security.authentication` is `kerberos` there, with the default credential cache +(`kinit`); `libgssapi_krb5` is loaded at runtime. PyIceberg's `hdfs.user` and +`hdfs.kerberos_ticket` are ignored with a warning. + +Unlike Hadoop, a portless authority is never resolved from `$HADOOP_CONF_DIR` or dialed on +port 8020: declare it with `hdfs.name-node.`.