diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 49eafc6e3a1..5c745a7b479 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -468,6 +468,7 @@ jobs: org.apache.comet.CometVariantProjectionSuite org.apache.spark.sql.CometVariantShreddingSuite org.apache.comet.CometIcebergNativeSuite + org.apache.comet.CometIcebergHdfsSuite org.apache.comet.CometIcebergEncryptionSuite org.apache.comet.CometIcebergRewriteActionSuite org.apache.comet.CometIcebergWriteActionSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 22bd6251dcf..0919b9c617e 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -172,6 +172,7 @@ jobs: org.apache.comet.CometVariantProjectionSuite org.apache.spark.sql.CometVariantShreddingSuite org.apache.comet.CometIcebergNativeSuite + org.apache.comet.CometIcebergHdfsSuite org.apache.comet.CometIcebergEncryptionSuite org.apache.comet.CometIcebergRewriteActionSuite org.apache.comet.CometIcebergWriteActionSuite diff --git a/docs/source/contributor-guide/roadmap.md b/docs/source/contributor-guide/roadmap.md index d318c07a539..927d0da9d93 100644 --- a/docs/source/contributor-guide/roadmap.md +++ b/docs/source/contributor-guide/roadmap.md @@ -67,10 +67,7 @@ would remove JVM round-trips beyond those the lambda work above addresses. Comet's native Iceberg scans read V3 tables, including encrypted tables ([#4991]) and tables with deletion vectors ([#5853]). We want to add the remaining V3 features so that these scans don't fall back to Spark: row lineage metadata columns, column default values, and the new V3 types (`variant`, `geometry`, `geography`, and `unknown`). -The work is tracked in [#3376], and upstream `iceberg-rust` support in [iceberg-rust #2411]. Native Iceberg scans -also don't support HDFS-backed tables today: Comet's native Iceberg storage layer handles only local files, S3 and -S3-compatible stores, GCS, and OSS, and would need an HDFS `StorageFactory` upstream in `iceberg-storage-opendal`. -We're scoping what that work would take. +The work is tracked in [#3376], and upstream `iceberg-rust` support in [iceberg-rust #2411]. [#3376]: https://github.com/apache/datafusion-comet/issues/3376 [#4991]: https://github.com/apache/datafusion-comet/pull/4991 diff --git a/docs/source/contributor-guide/s3-credential-provider-design.md b/docs/source/contributor-guide/s3-credential-provider-design.md index d4fa3e1baa9..91e4c37f15b 100644 --- a/docs/source/contributor-guide/s3-credential-provider-design.md +++ b/docs/source/contributor-guide/s3-credential-provider-design.md @@ -142,7 +142,7 @@ The 5-minute fallback on the Iceberg path bounds how long reqsign reuses a crede ## Property-bag handling on the Iceberg path -The full unfiltered FileIO property bag crosses JNI as `catalog_properties`. The storage-prefix filter (`s3.`/`gcs.`/`adls.`/`client.`/`opendal.`) is applied native-side in `iceberg_common.rs::build_file_io` immediately before `FileIOBuilder.with_prop`. This means the bridge sees `credentials.uri`, OAuth tokens, and any vendor-custom keys with no parallel field on the operator and no driver-side broadcast. Vendors set their own keys on the catalog config and read them back inside `initialize(Map)`. +The full unfiltered FileIO property bag crosses JNI as `catalog_properties`. The storage-prefix filter (`s3.`/`gcs.`/`adls.`/`client.`/`opendal.`/`hdfs.`/`hadoop.`) is applied native-side in `iceberg_common.rs::build_file_io` immediately before `FileIOBuilder.with_prop`. This means the bridge sees `credentials.uri`, OAuth tokens, and any vendor-custom keys with no parallel field on the operator and no driver-side broadcast. Vendors set their own keys on the catalog config and read them back inside `initialize(Map)`. `IcebergScanExec` derives a redacting `Debug`, and the `FileIO` cache key's `Debug` omits the property bag, so plan dumps and tracing do not leak it. diff --git a/docs/source/user-guide/latest/datasources.md b/docs/source/user-guide/latest/datasources.md index f351aff5b6f..6d505a67630 100644 --- a/docs/source/user-guide/latest/datasources.md +++ b/docs/source/user-guide/latest/datasources.md @@ -117,8 +117,9 @@ attaches its own worker threads, so a worker that has read from HDFS can crash t the HDFS read itself, typically while an unrelated query is running. ``` -Native Iceberg scans do not support HDFS-backed tables; those scans fall back to Spark. See the -[Comet and Iceberg Guide](iceberg.md). +Native Iceberg scans and writes on `hdfs://` tables do not go through `libhdfs`. They use iceberg-rust's +own pure-Rust HDFS client, which is part of every Comet build and is configured separately from the +settings on this page. See [Object store configuration (HDFS)](iceberg.md#object-store-configuration-hdfs). ### Building Comet with HDFS support diff --git a/docs/source/user-guide/latest/iceberg-writes.md b/docs/source/user-guide/latest/iceberg-writes.md index a04f6be61cf..4423c4723f9 100644 --- a/docs/source/user-guide/latest/iceberg-writes.md +++ b/docs/source/user-guide/latest/iceberg-writes.md @@ -199,7 +199,7 @@ A write is eligible only when ALL of the following hold: | `write.metadata.metrics.*` | any value (manifest metrics are re-derived on the JVM with Iceberg's own logic) | | `write.spark.fanout.enabled` | any value (the native writer implements both clustered and fanout modes) | | `write.target-file-size-bytes` | any value (the two writers can choose different roll points; see accepted divergences) | -| data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs`, matched case-sensitively (`S3://` falls back). `s3`, `s3a` and `gs` need a bucket in the authority (`s3://bucket/...`), so a hostless form such as `s3:/bucket/key` falls back. `gs` only when the `FileIO` opening the data location is a `GCSFileIO`; see below | +| data location URI scheme | `file`, `memory`, `s3`, `s3a`, `gs`, `hdfs`, matched case-sensitively (`S3://` falls back). `s3`, `s3a`, `gs` and `hdfs` need an authority (`s3://bucket/...`, `hdfs://namenode:8020/...`), so a hostless form such as `s3:/bucket/key` or `hdfs:///path` falls back. `gs` only when the `FileIO` opening the data location is a `GCSFileIO`, and `hdfs` only when the native HDFS client can reach the NameNode; see below | | resolved `table.locationProvider()` | Iceberg's built-in `DefaultLocationProvider` | | Hadoop S3A settings for an `s3` / `s3a` data location | only `fs.s3a.access.key`, `secret.key`, `session.token`, `endpoint`, `endpoint.region`, and `path.style.access`, including their `fs.s3a.bucket..*` forms; any other effective `fs.s3a.*` setting falls back | | Iceberg `FileIO` S3 settings for an `s3` / `s3a` data location | the S3 endpoint, region, static/session credentials, path-style, SSE (`none`, `s3`, `kms`, or `custom`; not `dsse-kms`), assume-role, anonymous/config-chain settings parsed by the pinned iceberg-rust version, plus Comet's credential-provider class and built-in web-identity properties. When a custom provider is configured, its vendor-owned `s3.*` / `client.*` properties are also forwarded; unsupported Iceberg-defined S3 settings still fall back | @@ -232,6 +232,20 @@ could resolve a different storage identity or endpoint than the JVM writer would combination falls back; a `GCSFileIO` carries its `gcs.*` settings in `FileIO.properties()`, which are forwarded. +An `hdfs` data location is checked at planning for whether the native HDFS client could reach its +NameNode. A location with no authority (`hdfs:///...`) falls back, even when `hdfs.name-node`, +`hdfs.host`, `hdfs.port` or `hadoop.fs.defaultFS` is set. A catalog `hdfs.name-node` or +`hdfs.name-node.` entry that is not `host:port` also falls back. So does a portless +authority (a nameservice) that neither the Hadoop configuration of the table's `FileIO` +(`dfs.ha.namenodes.` and a `host:port` `dfs.namenode.rpc-address..` +for each NameNode), which is the one the native writer's declaration is derived from, nor the +catalog (`hdfs.name-node.`) declares, as do the other HDFS cases listed in the +section linked below. A portless authority +that the Hadoop configuration does not name as a nameservice is a plain host on port 8020, as for +the JVM client. Each is reported as a fall-back reason like any other, rather than surfacing as a +failed task. See [Object store configuration (HDFS)](iceberg.md#object-store-configuration-hdfs) +for how the NameNode list is derived. + For an `s3` or `s3a` data location, the gate also inspects both the table FileIO's effective Hadoop configuration and `table.io().properties()`. These are separate allowlists because Hadoop S3A keys are translated before they reach iceberg-rust, while Iceberg `FileIO` keys are forwarded diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 4c3abe00216..ce1a3e857ee 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -118,8 +118,9 @@ safe partial pruning is tracked in [#5883](https://github.com/apache/datafusion- - Local filesystem - S3-compatible storage (AWS S3, MinIO) - Google Cloud Storage (`gs`) and Alibaba Cloud OSS (`oss`) - -HDFS-backed tables are not supported by the native Iceberg reader and fall back to Spark. +- HDFS (`hdfs`), through iceberg-rust's own HDFS client. See + [Object store configuration (HDFS)](#object-store-configuration-hdfs) for NameNode + configuration and the cases that fall back to Spark ### REST Catalog @@ -174,6 +175,43 @@ For a custom S3-compatible endpoint, configure the catalog with the endpoint, pa These `s3.*` storage properties are not specific to the Hive catalog shown here. When `s3.access-key-id` / `s3.secret-access-key` are omitted, credentials come from the standard AWS chain (environment variables, instance profiles, and so on). The region is not auto-detected: when neither the catalog (`client.region` or `s3.region`) nor the executor environment (`AWS_REGION` or `AWS_DEFAULT_REGION`) supplies one, Comet uses `us-east-1`, so set it for AWS buckets in any other region. If your REST catalog vends temporary credentials, the native reader does not consume them automatically, and wiring that requires the credential provider bridge. See Iceberg's [S3 FileIO](https://iceberg.apache.org/docs/latest/aws/#s3-fileio) docs for the full property list, and [S3 Credential Providers](s3-credential-providers.md) for vended or per-request credentials. +### Object store configuration (HDFS) + +`hdfs://` tables are read and written through iceberg-rust's `hdfs-native` backend, a pure-Rust HDFS RPC client. This is **not** the libhdfs/JNI client that the plain-Parquet native scan uses for `spark.hadoop.fs.comet.libhdfs.schemes`: the two clients live in the same process but connect independently, so an Iceberg table and a plain Parquet file on the same cluster each open their own connections. The Rust client still reads `core-site.xml` / `hdfs-site.xml` from `$HADOOP_CONF_DIR` (or `$HADOOP_HOME/etc/hadoop`) on each executor. It does not reuse the JVM's `UserGroupInformation` or the delegation tokens Spark obtains: it authenticates as the executor process, through the system `libgssapi_krb5` and the default Kerberos ticket cache, or otherwise as `HADOOP_USER_NAME`. It does read a token file named by `HADOOP_TOKEN_FILE_LOCATION`, but it looks HDFS delegation tokens up under the service `ha-hdfs:nameservice` of its own synthetic nameservice (see below), so a token issued for the cluster is not expected to match. Comet's tests do not cover a secured cluster. + +The Rust client finds the NameNode from the authority of each location. An authority with a port (`hdfs://nn.example.com:8020/...`) is dialed as is, so a single-NameNode cluster addressed that way needs no configuration. An authority without a port (`hdfs://nameservice1/...`) is a logical nameservice, which the client resolves only through a declaration in the properties it is given: `hdfs.name-node.`, a comma-separated `host:port` list, or Hadoop's `dfs.ha.namenodes.` and `dfs.namenode.rpc-address..` keys passed as `hadoop.*` properties. It does not look the nameservice up in `$HADOOP_CONF_DIR`, and it has no default port. The global `hdfs.name-node` property serves only locations with no authority, so it does not apply to a location that names a NameNode or nameservice. + +Comet writes this declaration for you, as `hdfs.name-node.`, on the driver, from the Hadoop configuration of the table's `FileIO` (which carries the catalog's `hadoop.*` overrides, as the JVM reader sees them), or from the session Hadoop configuration when the `FileIO` has none. The authority is that of the table's data files, which Iceberg allows to differ from the metadata location, or that of the data location for a write. For an HA nameservice, Comet reads `dfs.ha.namenodes.` and each `dfs.namenode.rpc-address..` and declares the same failover list the JVM client would use. Each `rpc-address` must be `host:port`; the JVM client rejects one without a port as well. The list is all-or-nothing: if any NameNode named in `dfs.ha.namenodes.` has no `host:port` `rpc-address`, Comet declares nothing rather than a partial failover list, and the scan or write falls back (see below). A portless authority that the Hadoop configuration does not name as a nameservice (through `dfs.nameservices` or `dfs.ha.namenodes.`) is a plain host, which Comet declares on port 8020, Hadoop's default NameNode RPC port, as the JVM client dials it. Nothing needs to be set as long as the standard HDFS client configuration is on the driver's classpath. To override the derived list, or to declare a nameservice the Hadoop configuration does not resolve (a federated nameservice without HA, for example), set the property on the catalog: + +```shell + --conf spark.sql.catalog.hdfs_cat=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.hdfs_cat.type=hadoop \ + --conf spark.sql.catalog.hdfs_cat.warehouse=hdfs://nameservice1/warehouse \ + --conf spark.sql.catalog.hdfs_cat.hdfs.name-node.nameservice1=nn1.example.com:8020,nn2.example.com:8020 +``` + +An explicit catalog value always wins over the one derived from the Hadoop configuration. It reaches Comet through the table's `FileIO` properties, so it takes effect only with a catalog that initializes its `FileIO` with the catalog properties: the Hadoop, JDBC and REST catalogs do, and the Hive catalog does only when `io-impl` is set. Every entry must be `host:port` (an IPv6 literal in brackets, such as `[::1]:8020`). The `hdfs://` prefix is optional, and spaces around an entry and a trailing `/` are tolerated. Comet does not add a default port to an entry you write yourself. Hadoop's own keys can be set the same way, as `hadoop.dfs.ha.namenodes.` and `hadoop.dfs.namenode.rpc-address..`, and they win over `hdfs.name-node.`. + +Other HDFS client settings can be forwarded with `hadoop.`-prefixed catalog properties (for example `spark.sql.catalog.hdfs_cat.hadoop.dfs.client.use.datanode.hostname=true`), which reach Comet through the same catalogs and override the values loaded from `$HADOOP_CONF_DIR`. The client connects through a nameservice of its own rather than the cluster's, so settings keyed by the cluster's nameservice name, such as `dfs.client.failover.proxy.provider.` or `dfs.client.failover.random.order.`, do not apply to it. + +The following cases fall back to Spark at planning, instead of failing on an executor: + +- A location with no authority (`hdfs:///warehouse/...`) falls back for both reads and writes, even when `hdfs.name-node`, `hdfs.host`, `hdfs.port` or `hadoop.fs.defaultFS` is set. Comet requires the NameNode or nameservice in the location itself. +- A catalog `hdfs.name-node` or `hdfs.name-node.` entry that is not `host:port`, such as one without a port, falls back, and the reason names the property and the entry. This holds for every such property the catalog sets, including one for another nameservice, because iceberg-rust parses them all when it opens the storage. +- A portless authority that neither the Hadoop configuration nor the catalog declares falls back. Typically the Hadoop configuration names the nameservice, but a `dfs.namenode.rpc-address..` entry is missing or has no port, or the nameservice has no `dfs.ha.namenodes.`: fix the Hadoop configuration, or set `hdfs.name-node.` on the catalog. +- An authority with userinfo (`hdfs://user@nn.example.com:8020/...`), or with a port that is not a number from 1 to 65535, falls back. +- A read whose data and delete files carry more than one `hdfs://` authority falls back, because Comet declares the NameNodes of a single authority per scan. Authorities are compared as written, so one cluster under two spellings counts as two. +- A nameservice whose NameNodes are found through DNS (`dfs.client.failover.resolve-needed.=true`) falls back: the native client dials each `rpc-address` once and does not expand a name to its addresses, so it would keep a single failover target. +- A portless authority that is not a plain ASCII host or nameservice name (an IPv6 literal without a port, or a non-ASCII name) falls back, because iceberg-rust looks such names up in a rewritten form. +- An `hdfs.port` that is not a port, an `hdfs.host` that does not form `host:port` with it, or a bare `hadoop.` property falls back, because iceberg-rust rejects them for every path when it opens the storage. +- For the same reason, an incomplete nameservice declaration passed as `hadoop.dfs.ha.namenodes.` falls back, even for a location in another nameservice: every NameNode it lists needs a `host:port` `hadoop.dfs.namenode.rpc-address..`. +- A catalog that forwards `hadoop.fs.defaultFS` falls back unless that value is an HDFS `host:port` or the catalog also sets `hdfs.name-node`. A logical default filesystem such as `hdfs://nameservice1` is rejected by iceberg-rust even when the nameservice is declared. +- A catalog that declares a nameservice named `nameservice` (as `hdfs.name-node.nameservice` or `hadoop.dfs.ha.namenodes.nameservice`) falls back for any other location, because that is the name of the client's own synthetic nameservice and the declaration would replace its NameNodes. + +A host, with or without a port, is not validated, so a mistyped host still fails when a task first opens a file. + +Failover follows a NameNode that refuses connections or answers as a standby, which the HA test covers before and after a failover and with a NameNode shut down. Two limits of the native client remain. It has no NameNode connect or RPC timeout, so a NameNode that stops answering without refusing connections can stall a read or write rather than fail over. And it counts every failover against `dfs.client.failover.max.attempts` but sleeps only once per round over the list, so with more than two NameNodes (observers included) it waits less time than the JVM client for a new active. To see why a table fell back, set `spark.comet.explain.fallback.enabled=true`: the driver log then lists the reason each stage could not run in Comet (see [Understanding Comet Plans](understanding-comet-plans.md)). + ### Current limitations The following scenarios will fall back to the JVM Iceberg reader: @@ -188,6 +226,13 @@ The following scenarios will fall back to the JVM Iceberg reader: - Tables backed by Avro or ORC data files (only Parquet is accelerated) - Scans whose data or delete files span more than one S3 bucket (the native reader uses one object-store configuration per scan) +- Scans whose data or delete files span more than one HDFS authority (Comet declares the + NameNodes of one authority per scan) +- HDFS locations the native client could not reach: no authority (`hdfs:///...`), an + `hdfs.name-node` or `hdfs.name-node.` entry that is not `host:port`, or a + nameservice that neither the Hadoop configuration nor the catalog declares with `host:port` + NameNodes, and the other HDFS cases listed in + [Object store configuration (HDFS)](#object-store-configuration-hdfs) - Tables partitioned on `BINARY` or `DECIMAL` (with precision >28) columns - Scans with residual filters using `truncate`, `bucket`, `year`, `month`, `day`, or `hour` transform functions (partition pruning still works, but row-level filtering of these @@ -260,3 +305,7 @@ the expression fall back to Spark. The native Iceberg reader populates Spark's task-level `inputMetrics.bytesRead` (visible in the Spark UI Stages tab) using the `bytes_read` counter from iceberg-rust's `ScanMetrics`. This counter includes bytes read from both data files and delete files. The scan's SQL metrics, including Iceberg's planning counters, are listed in the [Metrics Guide](metrics.md#cometicebergnativescan). Iceberg Java does not explicitly report `bytesRead` to Spark's task input metrics. On the iceberg Java path, any `bytesRead` value comes from Hadoop's filesystem-level I/O counters, not from Iceberg itself. Because Comet's native reader and the Hadoop filesystem use different counting mechanisms, the exact byte counts will differ between the two paths. + +The task-level `inputMetrics.recordsRead` is the scan's `number of output rows`. iceberg-rust applies the residual predicate Comet hands it as a row filter inside the scan, so both count the rows that pass it. They can therefore be lower than the `BatchScan` figures on the Iceberg Java path, where every row leaves the scan and is filtered by the `Filter` above it. + +Iceberg's `number of row deletes applied` is not reported for the native scan. It counts deletes applied by Iceberg Java's reader, and iceberg-rust's `ScanMetrics` exposes bytes read only, so Comet omits it instead of showing a constant 0. diff --git a/native/Cargo.lock b/native/Cargo.lock index e4d2aac7e1d..6c67aa36759 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -34,10 +34,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] +[[package]] +name = "aes" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.1", +] + [[package]] name = "aes-gcm" version = "0.10.3" @@ -45,9 +56,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead", - "aes", - "cipher", - "ctr", + "aes 0.8.4", + "cipher 0.4.4", + "ctr 0.9.2", "ghash", "subtle", ] @@ -75,6 +86,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "aligned-vec" version = "0.6.4" @@ -178,26 +195,26 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "apache-avro" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" dependencies = [ "bigdecimal", "bon", "crc32fast", - "digest 0.10.7", + "digest 0.11.3", "log", - "miniz_oxide 0.8.9", + "miniz_oxide 0.9.1", "num-bigint 0.4.8", + "ouroboros", "quad-rand", - "rand 0.9.5", + "rand 0.10.2", "regex-lite", "serde", "serde_bytes", "serde_json", "snap", - "strum", - "strum_macros", + "strum 0.28.0", "thiserror 2.0.20", "uuid", "zstd 0.13.3", @@ -1165,6 +1182,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +dependencies = [ + "hybrid-array", +] + [[package]] name = "blocking" version = "1.7.0" @@ -1296,7 +1322,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "cbc" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +dependencies = [ + "cipher 0.5.2", ] [[package]] @@ -1398,7 +1433,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout 0.2.2", ] [[package]] @@ -1429,7 +1475,7 @@ version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 3.0.5", @@ -1606,6 +1652,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -1624,6 +1676,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc-fast" version = "1.10.0" @@ -1784,7 +1851,16 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", +] + +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher 0.5.2", ] [[package]] @@ -2019,7 +2095,7 @@ dependencies = [ "itertools 0.15.0", "jni 0.22.4", "lazy_static", - "libloading", + "libloading 0.8.9", "log", "log4rs", "mimalloc", @@ -2875,6 +2951,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "des" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916a94e407b54f9034d71dd748234cd1e516ced6284009906ae246f177eafe5a" +dependencies = [ + "cipher 0.5.2", +] + [[package]] name = "destructure_traitobject" version = "0.2.0" @@ -2925,6 +3010,18 @@ dependencies = [ "const-random", ] +[[package]] +name = "dns-lookup" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" +dependencies = [ + "cfg-if", + "libc", + "socket2", + "windows-sys 0.60.2", +] + [[package]] name = "dunce" version = "1.0.5" @@ -3213,6 +3310,34 @@ dependencies = [ "slab", ] +[[package]] +name = "g2gen" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a7e0eb46f83a20260b850117d204366674e85d3a908d90865c78df9a6b1dfc" +dependencies = [ + "g2poly", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "g2p" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "539e2644c030d3bf4cd208cb842d2ce2f80e82e6e8472390bcef83ceba0d80ad" +dependencies = [ + "g2gen", + "g2poly", +] + +[[package]] +name = "g2poly" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312d2295c7302019c395cfb90dacd00a82a2eabd700429bba9c7a3f38dbbe11b" + [[package]] name = "generic-array" version = "0.14.7" @@ -3232,7 +3357,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -3359,6 +3484,47 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hdfs-native" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd181084003308224efddf737417832186839ce2882d2468ddad0114cfb3551" +dependencies = [ + "aes 0.9.3", + "base64 0.22.1", + "bitflags 2.13.2", + "bumpalo", + "bytes", + "cbc 0.2.1", + "chrono", + "cipher 0.5.2", + "crc", + "ctr 0.10.1", + "des", + "dns-lookup", + "futures", + "g2p", + "hex", + "hmac 0.13.0", + "libc", + "libloading 0.9.0", + "log", + "md-5 0.11.0", + "num-traits", + "once_cell", + "prost", + "prost-types", + "rand 0.10.2", + "regex", + "roxmltree", + "socket2", + "thiserror 2.0.20", + "tokio", + "url", + "uuid", + "whoami", +] + [[package]] name = "hdfs-sys" version = "0.3.1" @@ -3383,6 +3549,12 @@ dependencies = [ "log", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -3582,7 +3754,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=af1da4c5bc86178c38c1c6db0578bdd9fb7021e3#af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" +source = "git+https://github.com/mixermt/iceberg-rust?rev=0145ca6b6bce85a0cd52dda2636f0402e55969f8#0145ca6b6bce85a0cd52dda2636f0402e55969f8" dependencies = [ "aes-gcm", "anyhow", @@ -3626,7 +3798,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_with", - "strum", + "strum 0.27.2", "tokio", "tracing", "typed-builder", @@ -3640,7 +3812,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=af1da4c5bc86178c38c1c6db0578bdd9fb7021e3#af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" +source = "git+https://github.com/mixermt/iceberg-rust?rev=0145ca6b6bce85a0cd52dda2636f0402e55969f8#0145ca6b6bce85a0cd52dda2636f0402e55969f8" dependencies = [ "proc-macro2", "quote", @@ -3650,7 +3822,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=af1da4c5bc86178c38c1c6db0578bdd9fb7021e3#af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" +source = "git+https://github.com/mixermt/iceberg-rust?rev=0145ca6b6bce85a0cd52dda2636f0402e55969f8#0145ca6b6bce85a0cd52dda2636f0402e55969f8" dependencies = [ "anyhow", "async-trait", @@ -3663,6 +3835,8 @@ dependencies = [ "reqsign-aws-v4", "reqsign-core", "serde", + "tokio", + "tracing", "typetag", "url", ] @@ -3824,10 +3998,20 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "block-padding", + "block-padding 0.3.3", "generic-array", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "block-padding 0.4.2", + "hybrid-array", +] + [[package]] name = "inventory" version = "0.3.24" @@ -3984,7 +4168,7 @@ dependencies = [ "java-locator", "jni-macros", "jni-sys 0.4.1", - "libloading", + "libloading 0.8.9", "log", "simd_cesu8", "thiserror 2.0.20", @@ -4151,6 +4335,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "liblzma" version = "0.4.8" @@ -4186,6 +4380,15 @@ dependencies = [ "cc", ] +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "link-section" version = "0.19.3" @@ -4359,7 +4562,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -4528,6 +4731,24 @@ dependencies = [ "libm", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "object" version = "0.37.3" @@ -4635,6 +4856,7 @@ dependencies = [ "opendal-service-fs", "opendal-service-gcs", "opendal-service-hdfs", + "opendal-service-hdfs-native", "opendal-service-oss", "opendal-service-s3", ] @@ -4803,6 +5025,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-hdfs-native" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebbf956ea9e64d70fc8d1f25381604c02528805c23bd55443b8e924b438222" +dependencies = [ + "bytes", + "futures", + "hdfs-native", + "log", + "opendal-core", + "serde", +] + [[package]] name = "opendal-service-oss" version = "0.58.2" @@ -4875,6 +5111,30 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + [[package]] name = "outref" version = "0.5.2" @@ -5126,8 +5386,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ - "aes", - "cbc", + "aes 0.8.4", + "cbc 0.1.2", "der", "pbkdf2", "scrypt", @@ -5297,6 +5557,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + [[package]] name = "procfs" version = "0.18.0" @@ -5337,7 +5610,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", @@ -5895,6 +6168,15 @@ dependencies = [ "byteorder", ] +[[package]] +name = "roxmltree" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] + [[package]] name = "rsa" version = "0.9.10" @@ -6054,7 +6336,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -6471,6 +6753,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "str_stack" version = "0.1.1" @@ -6489,7 +6777,16 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", ] [[package]] @@ -6498,7 +6795,19 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ - "heck", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.119", @@ -7108,6 +7417,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -7117,6 +7435,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.128" @@ -7227,6 +7554,19 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -7335,6 +7675,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -7368,13 +7717,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -7387,6 +7753,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -7399,6 +7771,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -7411,12 +7789,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -7429,6 +7819,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -7441,6 +7837,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -7453,6 +7855,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -7465,6 +7873,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -7493,6 +7907,12 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.3" diff --git a/native/Cargo.toml b/native/Cargo.toml index defbf41ae27..67052f0dcc0 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -82,8 +82,13 @@ aws-sdk-sts = { version = "1.114.0", default-features = false } # Pinned to a revision rather than a release because no published iceberg-rust # version is on our arrow: 0.10.1 requires arrow/parquet ^58. Move to a plain # version requirement once a release ships on arrow 59. Bump policy: #5645. -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "af1da4c5bc86178c38c1c6db0578bdd9fb7021e3" } -iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "af1da4c5bc86178c38c1c6db0578bdd9fb7021e3", features = ["opendal-memory", "opendal-fs", "opendal-s3", "opendal-gcs", "opendal-oss", "opendal-azdls"] } +# +# This branch pins the head of apache/iceberg-rust#3111 (HDFS via opendal services-hdfs-native) on +# the author's fork instead of an apache revision. The head contains the revision main pins +# (af1da4c5), the HDFS commits, and the newer apache/iceberg-rust main commits the PR has merged. +# Retarget at apache/iceberg-rust once #3111 merges. +iceberg = { git = "https://github.com/mixermt/iceberg-rust", rev = "0145ca6b6bce85a0cd52dda2636f0402e55969f8" } +iceberg-storage-opendal = { git = "https://github.com/mixermt/iceberg-rust", rev = "0145ca6b6bce85a0cd52dda2636f0402e55969f8", features = ["opendal-memory", "opendal-fs", "opendal-s3", "opendal-gcs", "opendal-oss", "opendal-azdls", "opendal-hdfs-native"] } reqsign-core = "3" [profile.release] diff --git a/native/core/src/execution/operators/iceberg_common.rs b/native/core/src/execution/operators/iceberg_common.rs index a17eb076e0d..2016e54de2d 100644 --- a/native/core/src/execution/operators/iceberg_common.rs +++ b/native/core/src/execution/operators/iceberg_common.rs @@ -46,7 +46,14 @@ const ICEBERG_PROVIDER_CLASS_PROPERTY: &str = "s3.comet.credential.provider.clas /// URI, OAuth tokens, credentials.uri, tenant-id, etc.) is kept upstream so /// `CometS3CredentialBridge` can read whatever the vendor needs. `opendal.` carries /// iceberg-storage-opendal's own settings, such as `opendal.io-timeout-ms`. -const STORAGE_PROPERTY_PREFIXES: &[&str] = &["s3.", "gcs.", "adls.", "client.", "opendal."]; +/// +/// `hdfs.` carries the NameNode declarations (`hdfs.name-node.`) and `hadoop.` the +/// HDFS client overrides; dropping them would leave an HA table's nameservice authority +/// undeclared, and iceberg-rust cannot resolve it (see +/// `CometIcebergNativeScan.hadoopToIcebergHdfsProperties`). +const STORAGE_PROPERTY_PREFIXES: &[&str] = &[ + "s3.", "gcs.", "adls.", "client.", "opendal.", "hdfs.", "hadoop.", +]; /// Pick an OpenDAL storage backend for a URI whose scheme `builtin_storage_schemes` lists for /// `access_mode`, or that is an opted-in S3-compliant alias written in lowercase; anything else is @@ -79,6 +86,10 @@ pub(crate) fn storage_factory_for( "memory" => Ok((Arc::new(OpenDalStorageFactory::Memory), true)), "gs" => Ok((Arc::new(OpenDalStorageFactory::Gcs), true)), "oss" => Ok((Arc::new(OpenDalStorageFactory::Oss), true)), + // iceberg-rust's pure-Rust `hdfs-native` backend -- NOT the libhdfs/JNI client the + // plain-Parquet path uses (`fs.comet.libhdfs.schemes`). Both link into the same + // `libcomet`, but they are separate clients with separate connections and Kerberos state. + "hdfs" => Ok((Arc::new(OpenDalStorageFactory::HdfsNative), true)), // s3, s3a, and any opted-in s3-compliant alias (e.g. blob) route to the S3 backend. Listed // last so the built-in backends above stay authoritative even if one of their schemes is // also named in `fs.comet.s3Compliant.schemes`. An alias additionally gets a wrapper that @@ -302,10 +313,13 @@ fn cached_file_io( /// mode; the JVM read and write gates load these lists over JNI. `memory` is write-only: an OpenDAL /// memory backend is a fresh empty in-process store the write path assembles manifests in, so a /// read finds nothing. `oss` is read-only: no `oss.*` property is forwarded and no test covers it. +/// `hdfs` is admitted for both: its NameNode declarations and client overrides travel as `hdfs.*` +/// and `hadoop.*` properties (`STORAGE_PROPERTY_PREFIXES`), and the JVM gates decline a location +/// whose NameNode the native client could not resolve. pub(crate) fn builtin_storage_schemes(access_mode: AccessMode) -> &'static [&'static str] { match access_mode { - AccessMode::Read => &["file", "gs", "oss", "s3", "s3a"], - AccessMode::Write => &["file", "memory", "gs", "s3", "s3a"], + AccessMode::Read => &["file", "gs", "hdfs", "oss", "s3", "s3a"], + AccessMode::Write => &["file", "memory", "gs", "hdfs", "s3", "s3a"], } } @@ -334,6 +348,18 @@ pub(crate) fn load_file_io( ) } +/// The catalog properties iceberg-rust's `FileIO` is given: those under a +/// [`STORAGE_PROPERTY_PREFIXES`] entry. The full unfiltered bag (catalog URI, OAuth tokens, +/// credentials.uri, tenant-id, etc.) is kept upstream so `CometS3CredentialBridge` can read +/// whatever the vendor needs. +fn storage_properties( + catalog_properties: &HashMap, +) -> impl Iterator { + catalog_properties + .iter() + .filter(|(key, _)| STORAGE_PROPERTY_PREFIXES.iter().any(|p| key.starts_with(p))) +} + /// Build a `FileIO` whose storage scheme is inferred from `reference_path` and whose properties /// come from the catalog. The reference path is the metadata location for reads or the data /// location for writes — anything that carries the right URI scheme. `catalog_name` is the @@ -353,13 +379,8 @@ fn build_file_io( )?; let mut file_io_builder = FileIOBuilder::new(factory); - // Narrow to storage-prefix keys before forwarding to iceberg-rust's FileIO. The full - // unfiltered bag (catalog URI, OAuth tokens, credentials.uri, tenant-id, etc.) is kept - // upstream so CometS3CredentialBridge can read whatever the vendor needs. - for (key, value) in catalog_properties { - if STORAGE_PROPERTY_PREFIXES.iter().any(|p| key.starts_with(p)) { - file_io_builder = file_io_builder.with_prop(key, value); - } + for (key, value) in storage_properties(catalog_properties) { + file_io_builder = file_io_builder.with_prop(key, value); } // Object-store's AmazonS3Builder defaults the SigV4 region to `us-east-1` when unset; @@ -610,6 +631,8 @@ fn is_iceberg_alias_scheme(scheme: &str, catalog_properties: &HashMap| { + FileIoCacheKey::new(props, table, "cat", AccessMode::Read) + }; + let base = props("hdfs://nn1:8020,hdfs://nn2:8020", "true"); + + // Control: the same properties always give the same key, so the inequalities below are + // down to the one property that changed. + assert!(key(&base).is_some()); + assert_eq!(key(&base), key(&base.clone())); + + // Only the `hdfs.name-node.nameservice1` declaration differs: another NameNode, another + // list, the same list reordered (which changes the failover order), and no list at all. + for other in [ + props("hdfs://nn1:8020,hdfs://nn3:8020", "true"), + props("hdfs://nn1:8020", "true"), + props("hdfs://nn2:8020,hdfs://nn1:8020", "true"), + ] { + assert_ne!(key(&base), key(&other), "{other:?}"); + } + let mut without_name_node = base.clone(); + without_name_node.remove("hdfs.name-node.nameservice1"); + assert_ne!(key(&base), key(&without_name_node)); + + // Only the `hadoop.dfs.client.failover.*` overrides differ: a changed value, an added + // override and a removed one. + let mut other_failover = base.clone(); + other_failover.insert( + "hadoop.dfs.client.failover.proxy.provider.nameservice1".to_string(), + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider".to_string(), + ); + let mut without_failover = base.clone(); + without_failover.remove("hadoop.dfs.client.failover.random.order"); + for other in [ + props("hdfs://nn1:8020,hdfs://nn2:8020", "false"), + other_failover, + without_failover, + ] { + assert_ne!(key(&base), key(&other), "{other:?}"); + } + } + #[test] fn cache_key_debug_does_not_leak_properties() { let props = HashMap::from([ @@ -948,7 +1034,9 @@ mod tests { #[test] fn unknown_scheme_is_rejected() { - let err = factory_result("hdfs://nn/db/table", AccessMode::Read).unwrap_err(); + // object_store recognizes abfss, but iceberg-rust's OpenDAL storage factory has no arm + // for it, so the JVM gate must decline rather than fail here at execution time. + let err = factory_result("abfss://c@acct/db/table", AccessMode::Read).unwrap_err(); assert!( err.contains("Unsupported storage scheme"), "unexpected error: {err}" @@ -977,6 +1065,164 @@ mod tests { assert!(has_explicit_s3_credentials(&blank)); } + #[test] + fn hdfs_scheme_resolves_for_both_modes() { + // Unlike `oss`, writes are admitted: `hdfs.`/`hadoop.` properties are forwarded, so + // nothing is silently dropped. + for mode in [AccessMode::Read, AccessMode::Write] { + assert!(factory_result("hdfs://nn:8020/warehouse/db/t", mode).is_ok()); + assert!(factory_result("hdfs://nameservice1/warehouse/db/t", mode).is_ok()); + } + } + + /// The HDFS properties the Scala side emits (`hdfs.name-node.` from + /// `hadoopToIcebergHdfsProperties`, `hadoop.*` from the catalog) next to the catalog identity + /// that must stay out of iceberg-rust's `FileIO`. + fn hdfs_catalog_properties() -> HashMap { + HashMap::from([ + ( + "hdfs.name-node.nameservice1".to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string(), + ), + ( + "hadoop.dfs.client.failover.random.order".to_string(), + "true".to_string(), + ), + ("uri".to_string(), "thrift://metastore:9083".to_string()), + ( + "warehouse".to_string(), + "hdfs://nameservice1/wh".to_string(), + ), + ("token".to_string(), "oauth-token".to_string()), + ("credential".to_string(), "client:secret".to_string()), + // The prefixes end in a dot: a bare key or a longer word is not under them. + ("hdfs".to_string(), "bare".to_string()), + ("hadoopish.key".to_string(), "lookalike".to_string()), + ]) + } + + #[test] + fn storage_properties_keep_hdfs_and_hadoop_and_drop_catalog_identity() { + let props = hdfs_catalog_properties(); + + let forwarded: BTreeMap<&str, &str> = storage_properties(&props) + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect(); + + // If the narrowing drops these, an HA table connects to its nameservice as if it were a + // host and fails only once a task opens a file. + assert_eq!( + forwarded, + BTreeMap::from([ + ("hadoop.dfs.client.failover.random.order", "true"), + ( + "hdfs.name-node.nameservice1", + "hdfs://nn1:8020,hdfs://nn2:8020" + ), + ]) + ); + } + + /// `build_file_io` has to use [`storage_properties`]: the properties a `FileIO` ends up with + /// are what the HDFS storage is later built from. `FileIOBuilder::build` is lazy and cannot + /// fail, so only the `FileIO`'s own config shows what was forwarded. + #[test] + fn hdfs_properties_reach_the_file_io() { + let (file_io, cacheable) = build_file_io( + &hdfs_catalog_properties(), + "hdfs://nameservice1/db/t", + "cat", + AccessMode::Read, + ) + .unwrap(); + + assert!(cacheable); + assert_eq!( + file_io.config().props(), + &HashMap::from([ + ( + "hdfs.name-node.nameservice1".to_string(), + "hdfs://nn1:8020,hdfs://nn2:8020".to_string() + ), + ( + "hadoop.dfs.client.failover.random.order".to_string(), + "true".to_string() + ), + ]) + ); + } + + /// Builds the real HDFS storage from one property the way a `FileIO` does on first use. + /// Parsing is all it does: an operator, and with it a NameNode connection, is only made by + /// the first storage call, so this needs neither a NameNode nor a runtime. The storage is + /// inspected through its serde form, so this is a contract test on the pinned iceberg-rust + /// (mixermt/iceberg-rust b9624086, apache/iceberg-rust#3111) and its config field names. + fn build_hdfs_storage(key: &str, value: &str) -> Result { + let config = + StorageConfig::from_props(HashMap::from([(key.to_string(), value.to_string())])); + let storage = OpenDalStorageFactory::HdfsNative + .build(&config) + .map_err(|e| e.to_string())?; + Ok(serde_json::to_value(&*storage).expect("the storage serializes")) + } + + /// One Hadoop option of the parsed storage config. + fn parsed_option<'a>(storage: &'a serde_json::Value, key: &str) -> Option<&'a str> { + storage + .pointer("/HdfsNative/config/options") + .and_then(|options| options.get(key)) + .and_then(serde_json::Value::as_str) + } + + /// The contract `CometIcebergNativeScan.hadoopToIcebergHdfsProperties` relies on: it declares + /// an HA nameservice as `hdfs.name-node.`, and iceberg-rust expands that into + /// Hadoop's own `dfs.ha.namenodes.` / `dfs.namenode.rpc-address..`, the keys its + /// resolver reads for a portless path authority. A portless entry is refused when the + /// storage is built, naming the key and the entry. + #[test] + fn hdfs_nameservice_declaration_is_parsed_by_the_real_storage() { + for input in [ + "hdfs://nn1:8020,hdfs://nn2:8020", + // Whitespace around entries, a trailing slash and a missing prefix are tolerated. + " hdfs://nn1:8020/ , nn2:8020 ", + ] { + let storage = build_hdfs_storage("hdfs.name-node.ns1", input) + .unwrap_or_else(|e| panic!("`{input}` should be accepted: {e}")); + assert_eq!( + parsed_option(&storage, "dfs.ha.namenodes.ns1"), + Some("nn0,nn1"), + "input `{input}`: {storage}" + ); + assert_eq!( + parsed_option(&storage, "dfs.namenode.rpc-address.ns1.nn0"), + Some("nn1:8020"), + "input `{input}`: {storage}" + ); + assert_eq!( + parsed_option(&storage, "dfs.namenode.rpc-address.ns1.nn1"), + Some("nn2:8020"), + "input `{input}`: {storage}" + ); + } + + // A portless entry would only fail at the first I/O, so it is refused here, and the + // error names the offending key and entry rather than the whole list. + let err = + build_hdfs_storage("hdfs.name-node.ns1", "hdfs://nn1:8020,hdfs://nn2").unwrap_err(); + assert!( + err.contains("hdfs.name-node.ns1") + && err.contains("hdfs://nn2") + && !err.contains("nn1"), + "unexpected error: {err}" + ); + // The global key keeps its own validation; Comet no longer emits it, but a catalog may. + let err = build_hdfs_storage("hdfs.name-node", "hdfs://ns1").unwrap_err(); + assert!( + err.contains("hdfs.name-node") && err.contains("hdfs://ns1"), + "unexpected error: {err}" + ); + } + #[test] fn listed_schemes_are_accepted_by_storage_factory() { for mode in [AccessMode::Read, AccessMode::Write] { @@ -996,8 +1242,9 @@ mod tests { #[test] fn unlisted_schemes_are_rejected_for_both_modes() { + // `hdfs` is listed on this branch (see `hdfs_scheme_resolves_for_both_modes`). let unlisted = [ - "hdfs", "abfs", "abfss", "wasb", "wasbs", "gcs", "http", "https", "azure", + "abfs", "abfss", "wasb", "wasbs", "gcs", "http", "https", "azure", ]; for mode in [AccessMode::Read, AccessMode::Write] { for scheme in unlisted { diff --git a/native/core/src/execution/operators/iceberg_partition_value.rs b/native/core/src/execution/operators/iceberg_partition_value.rs index 970297283a7..fd174281fcd 100644 --- a/native/core/src/execution/operators/iceberg_partition_value.rs +++ b/native/core/src/execution/operators/iceberg_partition_value.rs @@ -419,7 +419,7 @@ mod tests { ( Transform::Day, day([-366, -1, -2, -2]), - day([-365, -1, -1, -1]), + day([-365, -1, -2, -2]), ), ( Transform::Hour, @@ -438,10 +438,11 @@ mod tests { for (i, (transform, java, rust)) in cases.iter().enumerate() { let label = format!("{transform} of {}", source.data_type()); assert_eq!(&comet[i], java, "{label}"); - // The reason Comet computes these: iceberg-rust floors the first two rows, and its - // `day` moves the last two into 1969-12-31 (apache/iceberg-rust#3315). If this - // starts failing, iceberg-rust's transforms have changed and delegating needs - // another look. + // The reason Comet computes these: iceberg-rust floors the first two rows where + // iceberg-java places them by the second before. Its `day` used to move the last + // two into 1969-12-31 too (apache/iceberg-rust#3315, fixed by #3323, which the + // pinned revision carries). If this starts failing, iceberg-rust's transforms have + // changed and delegating needs another look. assert_eq!(&iceberg_rust[i], rust, "iceberg-rust's {label}"); } } diff --git a/native/core/src/execution/operators/iceberg_scan.rs b/native/core/src/execution/operators/iceberg_scan.rs index ad69cac4f03..86b1291818d 100644 --- a/native/core/src/execution/operators/iceberg_scan.rs +++ b/native/core/src/execution/operators/iceberg_scan.rs @@ -272,14 +272,14 @@ impl IcebergScanExec { // Delete-file sizes are never serialized, so they always arrive as 0. If we ever // trust manifest sizes (pending the unreleased apache/iceberg#12554 fix), skip // already-sized files here instead of asserting. - debug_assert_eq!(delete.file_size_in_bytes, 0); + debug_assert_eq!(delete.file_size_in_bytes(), 0); // A deletion vector is range-read from content_offset, and iceberg-rust consults // file_size_in_bytes only on the Parquet delete path. Statting the Puffin file // would be one HEAD per file per Spark partition for a value nothing reads. - if delete.file_format == DataFileFormat::Puffin { + if delete.file_format() == DataFileFormat::Puffin { continue; } - needed.insert(delete.file_path.clone()); + needed.insert(delete.file_path().to_string()); } } if needed.is_empty() { @@ -336,19 +336,37 @@ impl IcebergScanExec { let deletes = task .deletes() .iter() - .map(|delete| { - let mut delete = delete.clone(); - if let Some(&size) = size_map.get(&delete.file_path) { - delete.file_size_in_bytes = size; - } - delete + .map(|delete| match size_map.get(delete.file_path()) { + Some(&size) => Self::with_file_size(delete, size), + None => Ok(delete.clone()), }) - .collect::>(); + .collect::, Error>>()?; *task = Self::rebuild_task_with_deletes(task, deletes)?; } Ok(()) } + /// A copy of `delete` with its file size set. `FileScanTaskDeleteFile`'s fields are private + /// with no mutator, so the copy is built through its builder, which re-runs its validation. + fn with_file_size( + delete: &FileScanTaskDeleteFile, + file_size_in_bytes: u64, + ) -> Result { + FileScanTaskDeleteFile::builder() + .with_file_path(delete.file_path().to_string()) + .with_file_size_in_bytes(file_size_in_bytes) + .with_file_type(delete.file_type()) + .with_file_format(delete.file_format()) + .with_partition_spec_id(delete.partition_spec_id()) + .with_equality_ids(delete.equality_ids().map(<[i32]>::to_vec)) + .with_referenced_data_file(delete.referenced_data_file().map(str::to_string)) + .with_content_offset(delete.content_offset()) + .with_content_size_in_bytes(delete.content_size_in_bytes()) + .with_record_count(delete.record_count()) + .with_key_metadata(delete.key_metadata().map(Box::<[u8]>::from)) + .build() + } + /// Rebuilds a [`FileScanTask`] carrying a new set of delete files. /// /// `FileScanTask::deletes` is private with no mutator, so the only way to change a task's @@ -690,19 +708,14 @@ mod tests { } fn delete_file(path: &str) -> FileScanTaskDeleteFile { - FileScanTaskDeleteFile { - file_path: path.to_string(), - file_type: DataContentType::PositionDeletes, - file_format: DataFileFormat::Parquet, - file_size_in_bytes: 0, - partition_spec_id: 0, - equality_ids: None, - referenced_data_file: None, - content_offset: None, - content_size_in_bytes: None, - record_count: None, - key_metadata: None, - } + FileScanTaskDeleteFile::builder() + .with_file_path(path.to_string()) + .with_file_type(DataContentType::PositionDeletes) + .with_file_format(DataFileFormat::Parquet) + .with_file_size_in_bytes(0) + .with_partition_spec_id(0) + .build() + .unwrap() } // A delete file we cannot stat must fail the scan, not be silently read with a missing/0 size. @@ -720,7 +733,7 @@ mod tests { result.is_err(), "expected an error when a delete file cannot be statted" ); - assert_eq!(tasks[0].deletes()[0].file_size_in_bytes, 0); + assert_eq!(tasks[0].deletes()[0].file_size_in_bytes(), 0); } // The real on-disk size is filled in from the FileIO, replacing the 0 placeholder. @@ -738,7 +751,10 @@ mod tests { .await .unwrap(); - assert_eq!(tasks[0].deletes()[0].file_size_in_bytes, bytes.len() as u64); + assert_eq!( + tasks[0].deletes()[0].file_size_in_bytes(), + bytes.len() as u64 + ); } // A present-but-undersized delete file (0-byte object, truncated write, or a HEAD with no @@ -757,7 +773,7 @@ mod tests { result.is_err(), "expected an error when a delete file is below the Parquet footer minimum" ); - assert_eq!(tasks[0].deletes()[0].file_size_in_bytes, 0); + assert_eq!(tasks[0].deletes()[0].file_size_in_bytes(), 0); } // No deletes means no stats and no error. diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index ba9b08bd9da..69a97bfdcd1 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -620,6 +620,10 @@ impl Drop for AbortOnDrop { } // Dropped from a plain JVM thread (`releasePlan`): run the delete to completion so the // files are gone before the task reports its failure. + // + // A throwaway runtime is safe for HDFS too: the pinned iceberg-storage-opendal builds + // every `hdfs-native` client in a runtime of its own (`HDFS_NATIVE_RUNTIME`), so an + // operator first built here is not tied to this runtime's lifetime. Err(_) => match tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 96802ca805d..8482f074aff 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -4565,33 +4565,45 @@ fn parse_file_scan_tasks_from_common( )) })?; - Ok(iceberg::scan::FileScanTaskDeleteFile { + // iceberg-rust holds the deletion-vector coordinates unsigned and rejects a negative + // one when it reads a manifest entry; reject it here the same way. + let non_negative = |value: Option, what: &str| { + value.map(u64::try_from).transpose().map_err(|_| { + GeneralError(format!("Delete file '{}' has a negative {what}", file_path)) + }) + }; + let content_offset = non_negative(del.content_offset, "content offset")?; + let content_size_in_bytes = non_negative(del.content_size_in_bytes, "content size")?; + + iceberg::scan::FileScanTaskDeleteFile::builder() // Passed RAW, like `data_file_path` below (same exact-string delete-matching // constraint -- see there). - file_path, - file_type, - file_format, + .with_file_path(file_path) + .with_file_type(file_type) + .with_file_format(file_format) // Not serialized; filled in by IcebergScanExec::fill_delete_file_sizes. - file_size_in_bytes: 0, - partition_spec_id: del.partition_spec_id, - equality_ids: if del.equality_ids.is_empty() { + .with_file_size_in_bytes(0) + .with_partition_spec_id(del.partition_spec_id) + .with_equality_ids(if del.equality_ids.is_empty() { None } else { Some(del.equality_ids.clone()) - }, + }) // Deletion-vector coordinates, which the serde sets only when file_format is // PUFFIN. referenced_data_file names the data file the vector applies to; the other // two locate the deletion-vector-v1 blob in its Puffin file. file_format above is // the discriminator, since Iceberg also populates referencedDataFile on // file-scoped Parquet position deletes. - referenced_data_file: del.referenced_data_file.clone(), - content_offset: del.content_offset, - content_size_in_bytes: del.content_size_in_bytes, - record_count, + .with_referenced_data_file(del.referenced_data_file.clone()) + .with_content_offset(content_offset) + .with_content_size_in_bytes(content_size_in_bytes) + .with_record_count(record_count) // Plaintext StandardKeyMetadata forwarded verbatim from the JVM; decoded by // iceberg-rust with no KMS unwrap. None for unencrypted delete files. - key_metadata: del.key_metadata.clone().map(Vec::into_boxed_slice), - }) + .with_key_metadata(del.key_metadata.clone().map(Vec::into_boxed_slice)) + // Runs iceberg-rust's validation, which requires a deletion vector's coordinates. + .build() + .map_err(|e| GeneralError(format!("Invalid Iceberg delete file: {e}"))) }) .collect::, ExecutionError>>()?; diff --git a/pom.xml b/pom.xml index a020f0719f9..782078a8e90 100644 --- a/pom.xml +++ b/pom.xml @@ -84,7 +84,7 @@ under the License. 3.25.5 1.16.0 provided - 3.3.4 + 3.4.2 18.3.0 1.9.13 2.43.0 @@ -703,6 +703,7 @@ under the License. 2.12.17 2.12 3.4.3 + 3.3.4 3.4 3.3.4 1.13.1 @@ -723,6 +724,7 @@ under the License. 2.12.18 2.12 3.5.9 + 3.3.4 3.5 3.3.4 1.13.1 @@ -743,6 +745,7 @@ under the License. 2.13.16 2.13 4.0.4 + 3.4.1 4.0 1.15.2 4.13.10 @@ -765,6 +768,7 @@ under the License. 2.13.17 2.13 4.1.3 + 3.4.2 4.1 1.16.0 4.13.10 @@ -785,6 +789,7 @@ under the License. 2.13.18 2.13 4.2.0 + 3.5.0 4.2 1.17.0 4.13.10 diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index c7c9a31909d..117e5a705a2 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -545,11 +545,44 @@ case class CometScanRule(session: SparkSession) } val icebergDataBucket: Option[String] = taskValidation.dataFileBuckets.headOption + // The HDFS analogue of the multi-bucket check above: Comet declares the NameNodes of one + // HDFS authority per scan (`hadoopToIcebergHdfsProperties`), so a second portless + // authority would reach native undeclared. + if (taskValidation.dataFileHdfsAuthorities.size > 1) { + fallbackReasons += + "Iceberg scan reads data/delete files across multiple HDFS authorities " + + s"(${taskValidation.dataFileHdfsAuthorities.toSeq.sorted.mkString(", ")}); " + + "Comet declares the NameNodes of a single HDFS authority per scan" + return withFallbackReasons(scanExec, fallbackReasons.toSet) + } + // The DATA authority, which Iceberg allows to differ from the metadata location + // (`write.data.path`); None for an empty scan or a non-HDFS table. + val icebergDataHdfsAuthority: Option[String] = + taskValidation.dataFileHdfsAuthorities.headOption + // The location whose authority names the NameNode: the DATA authority when the tasks + // yielded one, else the metadata location. Both the HDFS properties and the gate below use + // it, so they cannot disagree. + def hdfsLocationUri(metadataUri: java.net.URI): java.net.URI = + icebergDataHdfsAuthority + .map(authority => new java.net.URI(s"hdfs://$authority/")) + .getOrElse(metadataUri) + // Extract all Iceberg metadata once using reflection. // If any required reflection fails, this returns None, and we fall back to Spark. // First get metadataLocation and catalogProperties which are needed by the factory. val tableOpt = IcebergReflection.getTable(scanExec.scan) + // The Hadoop configuration HDFS NameNodes are resolved from: the session's, overlaid + // with the table FileIO's, which carries the catalog's `hadoop.*` overrides the JVM reader + // honours. Only an HDFS location asks, so other tables never resolve their FileIO. + lazy val hdfsResolutionConf: Configuration = CometIcebergNativeScan.hdfsResolutionConf( + hadoopConf, + tableOpt.flatMap(table => + scala.util.Try(IcebergReflection.getFileIOHadoopConf(table)).toOption.flatten)) + def hdfsHadoopConf(location: java.net.URI): Configuration = + if (NativeConfig.lowerScheme(location).contains("hdfs")) hdfsResolutionConf + else hadoopConf + val metadataLocationOpt = tableOpt.flatMap { table => IcebergReflection.getMetadataLocation(table) } @@ -607,13 +640,19 @@ case class CometScanRule(session: SparkSession) .flatMap(IcebergReflection.getFileIOProperties) .getOrElse(Map.empty) + // Before `fileIOProperties` so an explicit catalog `hdfs.name-node.` wins. + val hdfsUri = hdfsLocationUri(effectiveUri) + val hadoopDerivedHdfsProperties = CometIcebergNativeScan + .hadoopToIcebergHdfsProperties(hdfsUri, hdfsHadoopConf(hdfsUri)) + // Iceberg reads the alias opt-in from catalog_properties (IcebergScanCommon has no // object_store_options), and hadoopToIcebergS3Properties drops non-fs.s3a keys, so // re-add it. load_file_io narrows catalog_properties to storage-prefix keys before // iceberg-rust's FileIO, so the alias key never reaches FileIO -- but it is still // handed, unfiltered, to CometS3CredentialBridge, so a custom credential provider sees // it. That is intended: the provider gets the full property bag. - val catalogProperties = hadoopDerivedProperties ++ fileIOProperties ++ + val catalogProperties = hadoopDerivedProperties ++ hadoopDerivedHdfsProperties ++ + fileIOProperties ++ hadoopS3Options .get(COMET_S3_COMPLIANT_SCHEMES_KEY) .map(COMET_S3_COMPLIANT_SCHEMES_KEY -> _) @@ -657,6 +696,22 @@ case class CometScanRule(session: SparkSession) false } + // The native HDFS client dials an authority with a port as is and resolves a portless one + // (a nameservice) only through its `hdfs.name-node.` declaration. Decline what + // it could not reach (authority-less locations, a NameNode entry that is not host:port, a + // nameservice that did not resolve) here, against the very catalog properties native + // receives, instead of failing a task on an executor. + val hdfsUri = hdfsLocationUri(metadataUri) + val hdfsNameNodeReachable = CometIcebergNativeScan.hdfsNameNodeFallbackReason( + hdfsUri, + hdfsHadoopConf(hdfsUri), + metadata.catalogProperties) match { + case Some(reason) => + fallbackReasons += reason + false + case None => true + } + // Now perform all validation using the pre-extracted metadata // Check if table uses a FileIO implementation compatible with iceberg-rust. // Comet's native reader bypasses Iceberg Java's FileIO entirely and reads through @@ -1042,7 +1097,8 @@ case class CometScanRule(session: SparkSession) if (schemaSupported && fileIOCompatible && formatVersionSupported && defaultValuesSupported && schemaTypesSupported && encryptionKeyLengthSupported && taskValidation.allParquet && allSupportedFilesystems && allLocationsOpenable && - metadataSchemeSupported && partitionTypesSupported && unifiedPartitionTypeSupported && + metadataSchemeSupported && hdfsNameNodeReachable && partitionTypesSupported && + unifiedPartitionTypeSupported && transformFunctionsSupported && deleteFileTypesSupported && dppSubqueriesSupported && nestedFieldsSupported) { CometBatchScanExec( @@ -1339,6 +1395,10 @@ object CometScanRule extends Logging { * The one exception is an opt-in S3-compliant alias, which the native reader opens by promoting * the bucket from the first path segment (`s3_blob_fs_support.rs`), so a hostless * `blob:///bucket/key.parquet` IS openable when it carries a promotable bucket segment. + * + * `hdfs` follows the general rule: the authority is the NameNode. A hostless `hdfs:///path` + * could be opened from a configured `hdfs.name-node`, but this gate runs before the catalog + * properties are assembled, so it declines rather than guess. */ private[rules] def hasOpenableAuthority(uri: URI, s3CompliantSchemes: Set[String]): Boolean = { val scheme = NativeConfig.lowerScheme(uri) @@ -1383,6 +1443,9 @@ object CometScanRule extends Logging { // Buckets the native FileIO must read (data + delete files), for the single-config check in // CometScanRule; only S3-family locations contribute (see NativeConfig.bucketForUri). val dataFileBuckets = mutable.Set[String]() + // Distinct `hdfs://` authorities across data and delete files, for the single-NameNode check + // in CometScanRule. + val dataFileHdfsAuthorities = mutable.Set[String]() // First data/delete location with a readable scheme but no URL host (see // hasOpenableAuthority); non-empty => decline. One example suffices for the message. var hostlessLocation: Option[String] = None @@ -1401,6 +1464,10 @@ object CometScanRule extends Logging { unsupportedSchemes += scheme } else if (!hasOpenableAuthority(uri, s3CompliantSchemes)) { if (hostlessLocation.isEmpty) hostlessLocation = Some(rawPath) + } else if (scheme == "hdfs") { + // RAW authority, not getHost, which answers null for a nameservice carrying an underscore + // and would quietly weaken the single-NameNode check. + Option(uri.getRawAuthority).filter(_.nonEmpty).foreach(dataFileHdfsAuthorities += _) } else { // bucketForUri yields None for non-S3-family URIs, so no scheme re-check is needed here. NativeConfig.bucketForUri(uri, s3CompliantSchemes).foreach(dataFileBuckets += _) @@ -1460,6 +1527,7 @@ object CometScanRule extends Logging { nonIdentityTransform, deleteFiles, dataFileBuckets.toSet, + dataFileHdfsAuthorities.toSet, hostlessLocation) } } @@ -1473,4 +1541,5 @@ case class IcebergTaskValidationResult( nonIdentityTransform: Option[String], deleteFiles: java.util.List[_], dataFileBuckets: Set[String], + dataFileHdfsAuthorities: Set[String], hostlessLocation: Option[String]) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala index 335f02ff034..135e68d9a21 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala @@ -608,6 +608,388 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit } } + /** + * Declares the NameNodes of an HDFS location's authority to iceberg-rust's `hdfs-native` + * backend, from the session Hadoop configuration. + * + * The pinned iceberg-rust (apache/iceberg-rust#3111) dials an authority that carries a port + * (`hdfs://nn.example.com:8020/...`) as is. An authority without one is a logical name that + * resolves only through its declaration: `hdfs.name-node.`, a comma-separated + * `host:port` list (`hdfs://` optional), or Hadoop's own `dfs.ha.namenodes.` keys + * passed as `hadoop.*`. The global `hdfs.name-node` serves authority-less paths only, and there + * is no default port. Without a declaration an HA table therefore fails on its first read or + * write, after the planner has committed to the native path. + * + * Resolution follows the JVM client: + * - an HA nameservice (`dfs.ha.namenodes.`) maps to every + * `dfs.namenode.rpc-address..`, all-or-nothing, since a partial list would turn a + * failover into an outage. Each must be `host:port`: the JVM rejects a portless + * rpc-address, so one leaves the nameservice unresolved; + * - a portless authority that is not a configured nameservice (not in `dfs.nameservices`) is + * a plain host, which the JVM dials on the default NameNode RPC port, 8020 + * (`DFSUtilClient.getNNAddress`); + * - an authority with a port, an unresolved nameservice, a nameservice whose NameNodes are + * found through DNS (`dfs.client.failover.resolve-needed.`) and an authority-less + * location yield nothing; [[hdfsNameNodeFallbackReason]] declines all but the first. + * + * Call sites order this before the catalog properties, so an explicit + * `spark.sql.catalog..hdfs.name-node.` still wins. + * + * @param uri + * the data (scan and write) location whose authority names the NameNode + * @param hadoopConf + * the configuration from [[hdfsResolutionConf]] + */ + def hadoopToIcebergHdfsProperties( + uri: java.net.URI, + hadoopConf: org.apache.hadoop.conf.Configuration): Map[String, String] = { + if (!NativeConfig.lowerScheme(uri).contains("hdfs")) return Map.empty + // The RAW authority, not `getHost`, which answers null for a nameservice carrying an + // underscore. + val authority = Option(uri.getRawAuthority).filter(_.nonEmpty).getOrElse(return Map.empty) + if (!isLogicalHdfsAuthority(authority)) return Map.empty + + val nnIds = haNameNodeIds(hadoopConf, authority) + val nameNodes = if (resolvesNameNodesThroughDns(hadoopConf, authority)) { + // The native client dials each rpc-address once and never expands a DNS name to its + // addresses, so declaring the names would silently drop the failover targets. + None + } else if (nnIds.nonEmpty) { + val endpoints = nnIds.flatMap { nnId => + Option(hadoopConf.getTrimmed(s"dfs.namenode.rpc-address.$authority.$nnId")) + .flatMap(hdfsNameNode) + } + if (endpoints.size == nnIds.size) Some(endpoints.mkString(",")) else None + } else if (isConfiguredNameservice(hadoopConf, authority)) { + None + } else { + hdfsNameNode(s"$authority:$DefaultNameNodeRpcPort") + } + nameNodes.map(nameNodeDeclarationKey(authority) -> _).toMap + } + + /** + * Why iceberg-rust's `hdfs-native` backend could not reach the NameNode of an HDFS location, or + * None when it can. Shared by the scan and write planners, which pass the catalog properties + * exactly as they hand them to native (the Hadoop-derived declaration first, an explicit + * catalog value winning), so the decision and the executed configuration agree. Declining here + * replaces a task failure on an executor with a fallback. + * + * It mirrors the pinned iceberg-rust's resolver (`hdfs_native_effective_name_node`): + * - an authority-less `hdfs:///...` location falls back even when `hdfs.name-node`, + * `hdfs.host`, `hdfs.port` or `hadoop.fs.defaultFS` is set: Comet does not rely on them; + * - every `hdfs.name-node` and `hdfs.name-node.` entry must be `host:port`, + * because iceberg-rust parses them all when it builds the storage, so one bad entry fails + * every read and write through it; + * - an authority with userinfo, port 0 or an otherwise malformed port is rejected; + * - an authority with a valid port is dialed as is; + * - a portless authority needs a declaration, `hdfs.name-node.` or + * `hadoop.dfs.ha.namenodes.` with a `host:port` rpc-address per NameNode; + * - the rest of the HDFS storage configuration must parse (`hdfs.host`, `hdfs.port`, no bare + * `hadoop.` key, every declared nameservice complete, and a forwarded `fs.defaultFS` an + * HDFS `host:port` unless the global `hdfs.name-node` is set), and no nameservice may be + * declared under the name opendal gives its own synthetic one, `nameservice`, unless it is + * the location's own. + * + * Known limits it cannot see: Kerberos delegation tokens (the native client presents the + * synthetic service `ha-hdfs:nameservice`) and NameNodes that stop answering without refusing + * connections (the native client has no NameNode connect or RPC timeout). + * + * Mirrors `hdfs_native_config_parse_with` and `hdfs_native_effective_name_node` in + * iceberg-storage-opendal at the pinned revision; re-check it when the pin moves. + * + * @param uri + * the location whose authority names the NameNode (the data location when known) + * @param hadoopConf + * the Hadoop configuration the declaration was derived from, used only to word the reason + * @param catalogProperties + * the effective catalog properties handed to native + * @return + * Some(reason) to decline the native path; always None for a non-hdfs location + */ + def hdfsNameNodeFallbackReason( + uri: java.net.URI, + hadoopConf: org.apache.hadoop.conf.Configuration, + catalogProperties: Map[String, String]): Option[String] = { + if (!NativeConfig.lowerScheme(uri).contains("hdfs")) return None + + val authority = Option(uri.getRawAuthority).filter(_.nonEmpty).getOrElse { + return Some( + s"authority-less hdfs location '$uri': Comet requires the NameNode or nameservice in " + + "the location itself, even when hdfs.name-node, hdfs.host, hdfs.port or " + + "hadoop.fs.defaultFS is set") + } + + // The Hadoop options iceberg-rust builds from the properties, declarations expanded. + lazy val options = hdfsNativeOptions(catalogProperties) + invalidNameNodeProperty(catalogProperties) + .orElse(invalidHdfsStorageProperty(catalogProperties)) + .orElse(invalidHdfsDeclaration(catalogProperties, options)) + .orElse(syntheticNameserviceCollision(options, authority)) + .orElse { + if (authority.contains("@")) { + Some( + s"HDFS location authority '$authority' carries userinfo, which Comet's native HDFS " + + "client does not support") + } else if (hdfsNameNode(authority).isDefined) { + None + } else if (!isLogicalHdfsAuthority(authority)) { + Some( + s"HDFS location authority '$authority' is neither host:port (port 1-65535) nor a " + + "nameservice name") + } else if (options.contains(s"dfs.ha.namenodes.$authority")) { + None + } else { + val source = if (resolvesNameNodesThroughDns(hadoopConf, authority)) { + "its NameNodes are found through DNS " + + s"(dfs.client.failover.resolve-needed.$authority), which the native client does " + + "not expand, so it would lose every failover target but one" + } else if (isConfiguredNameservice(hadoopConf, authority)) { + s"the Hadoop configuration declares nameservice '$authority' but Comet " + + s"could not resolve it: it needs dfs.ha.namenodes.$authority and a host:port " + + s"dfs.namenode.rpc-address.$authority. for every NameNode listed there" + } else { + s"no declaration for '$authority' was found" + } + Some(s"HDFS location authority '$authority' has no port, so Comet's native HDFS client " + + s"resolves it only through hdfs.name-node.$authority, and $source. Set the catalog " + + s"property hdfs.name-node.$authority=[,]; it reaches Comet " + + "only from catalogs that initialize their FileIO with the catalog properties (for " + + "example hadoop or REST catalogs)") + } + } + } + + /** + * The Hadoop configuration HDFS NameNodes are resolved from: the session's `dfs.*` settings + * overlaid with those of the table's FileIO when it has a Hadoop configuration. The FileIO's + * carry the catalog's `hadoop.*` overrides the JVM reader honours; the session's carry what was + * set after the catalog built its FileIO. A nameservice either one names must not be mistaken + * for a plain host on port 8020 because the other does not. + */ + def hdfsResolutionConf( + sessionConf: org.apache.hadoop.conf.Configuration, + fileIOConf: Option[org.apache.hadoop.conf.Configuration]) + : org.apache.hadoop.conf.Configuration = { + val resolution = new org.apache.hadoop.conf.Configuration(false) + (Seq(sessionConf) ++ fileIOConf).foreach { conf => + conf.getPropsWithPrefix("dfs.").asScala.foreach { case (key, value) => + resolution.set(s"dfs.$key", value) + } + } + resolution + } + + // iceberg-rust's global NameNode key; a `.` suffix declares that nameservice. + private val HdfsNameNodeKey = "hdfs.name-node" + + // The name opendal's `hdfs-native` service gives the HA nameservice it synthesizes for every + // client; forwarded options declaring a real nameservice of that name replace its NameNodes. + private val OpendalSyntheticNameservice = "nameservice" + + private def nameNodeDeclarationKey(nameservice: String): String = + s"$HdfsNameNodeKey.$nameservice" + + // Hadoop's DFS_NAMENODE_RPC_PORT_DEFAULT, which the JVM client applies to a portless + // `hdfs://host` URI (DFSUtilClient.getNNAddress). It does NOT apply it to a portless + // rpc-address, which it rejects. + private val DefaultNameNodeRpcPort = 8020 + + // An ASCII host name (letters, digits, '.', '-', '_') or IPv4 literal. Conservative on purpose: + // the url crate iceberg-rust parses with re-encodes non-ASCII and canonicalizes IPv6 hosts, so + // a declaration keyed by anything else could miss the name iceberg-rust looks up. + private val HdfsHostName = """[A-Za-z0-9._-]+""" + // `host:port`, the host also allowed as a bracketed IPv6 literal. + private val HdfsHostPortPattern = s"""($HdfsHostName|\\[[0-9A-Fa-f:.]+\\]):(\\d{1,5})""".r + // A portless authority, which iceberg-rust treats as a logical nameservice name. + private val HdfsLogicalAuthorityPattern = s"""($HdfsHostName)""".r + + /** + * One NameNode spelling as iceberg-rust's `hdfs_native_name_node` accepts it, normalized to + * `hdfs://host:port`: trimmed, a trailing `/` and an `hdfs://` prefix optional, port 1-65535, + * no userinfo or path. None for anything else, a logical name included. + */ + private def hdfsNameNode(entry: String): Option[String] = { + val hostPort = entry.trim.stripSuffix("/").stripPrefix("hdfs://") + hostPort match { + case _ if hostPort.contains("://") => None + case HdfsHostPortPattern(host, port) if port.toInt >= 1 && port.toInt <= 65535 => + Some(s"hdfs://$host:${port.toInt}") + case _ => None + } + } + + /** A portless authority, which iceberg-rust treats as a logical nameservice name. */ + private def isLogicalHdfsAuthority(authority: String): Boolean = + HdfsLogicalAuthorityPattern.pattern.matcher(authority).matches() + + private def resolvesNameNodesThroughDns( + hadoopConf: org.apache.hadoop.conf.Configuration, + nameservice: String): Boolean = + hadoopConf.getBoolean(s"dfs.client.failover.resolve-needed.$nameservice", false) + + /** + * The other HDFS settings iceberg-rust's parser rejects for every path when it builds the + * storage: a bare `hadoop.` key, an `hdfs.port` that is not a port, and an `hdfs.host` that + * with that port (default 8020) does not form `host:port`. + */ + private def invalidHdfsStorageProperty( + catalogProperties: Map[String, String]): Option[String] = { + def nonBlank(key: String) = catalogProperties.get(key).map(_.trim).filter(_.nonEmpty) + val port = nonBlank("hdfs.port") + if (catalogProperties.contains("hadoop.")) { + Some("the property 'hadoop.' names no Hadoop key") + } else if (port.exists(p => !p.matches("""\d{1,5}""") || p.toInt > 65535)) { + Some(s"hdfs.port '${port.get}' is not a port") + } else { + nonBlank("hdfs.host").flatMap { host => + val bracketed = if (host.contains(":") && !host.startsWith("[")) s"[$host]" else host + // Composed as iceberg-rust does, so a host that itself carries a scheme fails here too. + val composed = s"hdfs://$bracketed:${port.getOrElse(DefaultNameNodeRpcPort.toString)}" + if (hdfsNameNode(composed).isDefined) None + else Some(s"hdfs.host '$host' with hdfs.port does not form a host:port NameNode") + } + } + } + + /** A declaration of opendal's synthetic nameservice name for a location that is not it. */ + private def syntheticNameserviceCollision( + options: Map[String, String], + authority: String): Option[String] = { + val declared = options.contains(s"dfs.ha.namenodes.$OpendalSyntheticNameservice") + if (declared && authority != OpendalSyntheticNameservice) { + Some( + s"a nameservice named '$OpendalSyntheticNameservice' is declared, the name Comet's " + + "native HDFS client gives its own synthetic nameservice, so it would route " + + s"'$authority' to that nameservice's NameNodes") + } else { + None + } + } + + private def haNameNodeIds( + hadoopConf: org.apache.hadoop.conf.Configuration, + nameservice: String): Seq[String] = + Option(hadoopConf.getTrimmedStrings(s"dfs.ha.namenodes.$nameservice")) + .map(_.toSeq) + .getOrElse(Seq.empty) + .filter(_.nonEmpty) + + private def isConfiguredNameservice( + hadoopConf: org.apache.hadoop.conf.Configuration, + authority: String): Boolean = + hadoopConf.getTrimmedStringCollection("dfs.nameservices").contains(authority) || + haNameNodeIds(hadoopConf, authority).nonEmpty + + /** + * The first `hdfs.name-node` or `hdfs.name-node.` property iceberg-rust would + * reject when it parses the storage configuration, as a fallback reason. + */ + private def invalidNameNodeProperty(catalogProperties: Map[String, String]): Option[String] = + catalogProperties.toSeq + .sortBy(_._1) + .iterator + .flatMap { case (key, value) => + val isGlobal = key == HdfsNameNodeKey + if (!isGlobal && !key.startsWith(s"$HdfsNameNodeKey.")) { + None + } else { + val nameservice = key.stripPrefix(HdfsNameNodeKey).stripPrefix(".") + val entries = value.split(",").toSeq.map(_.trim).filter(_.nonEmpty) + if (!isGlobal && (nameservice.isEmpty || nameservice.exists(_.isWhitespace))) { + Some(s"$key does not name a nameservice") + } else if (!isGlobal && entries.isEmpty) { + Some(s"$key lists no NameNodes") + } else { + entries.find(hdfsNameNode(_).isEmpty).map { entry => + s"$key entry '$entry' is not host:port: Comet's native HDFS client needs every " + + "entry as host:port (for example nn1.example.com:8020) and fails at the first " + + "read or write otherwise" + } + } + } + } + .toSeq + .headOption + + /** + * The Hadoop options iceberg-rust builds from the catalog properties + * (`hdfs_native_config_parse_with`): the `hadoop.*` keys with the prefix stripped, then each + * well-formed `hdfs.name-node.` expanded into `dfs.ha.namenodes.` = `nn0,nn1,..` and a + * `dfs.namenode.rpc-address..nn` per entry, an explicit key winning over the expansion + * key by key. A malformed declaration is reported by [[invalidNameNodeProperty]] first. + */ + private def hdfsNativeOptions(catalogProperties: Map[String, String]): Map[String, String] = { + val options = mutable.Map[String, String]() + catalogProperties.foreach { case (key, value) => + if (key.startsWith("hadoop.") && key.length > "hadoop.".length) { + options(key.stripPrefix("hadoop.")) = value + } + } + catalogProperties.toSeq.sortBy(_._1).foreach { case (key, value) => + if (key.startsWith(s"$HdfsNameNodeKey.")) { + val nameservice = key.stripPrefix(s"$HdfsNameNodeKey.") + val entries = value.split(",").toSeq.map(_.trim).filter(_.nonEmpty).flatMap(hdfsNameNode) + if (nameservice.nonEmpty && entries.nonEmpty) { + val ids = entries.indices.map(i => s"nn$i") + options.getOrElseUpdate(s"dfs.ha.namenodes.$nameservice", ids.mkString(",")) + ids.zip(entries).foreach { case (id, entry) => + options.getOrElseUpdate( + s"dfs.namenode.rpc-address.$nameservice.$id", + entry.stripPrefix("hdfs://")) + } + } + } + } + options.toMap + } + + /** + * What iceberg-rust validates in the expanded options when it builds the storage, failing every + * path through it: each declared nameservice must list NameNodes, each with a `host:port` + * rpc-address, and a forwarded `fs.defaultFS` must be an HDFS `host:port` unless the global + * `hdfs.name-node` is set (a logical one is rejected even when declared). + */ + private def invalidHdfsDeclaration( + catalogProperties: Map[String, String], + options: Map[String, String]): Option[String] = { + val declarations = options.toSeq.sortBy(_._1).iterator.flatMap { case (key, ids) => + val nameservice = key.stripPrefix("dfs.ha.namenodes.") + if (!key.startsWith("dfs.ha.namenodes.") || nameservice.isEmpty) { + None + } else { + val nnIds = ids.split(",").toSeq.map(_.trim).filter(_.nonEmpty) + if (nnIds.isEmpty) { + Some(s"HDFS nameservice '$nameservice' is declared with no NameNodes") + } else { + nnIds + .find { nnId => + options + .get(s"dfs.namenode.rpc-address.$nameservice.$nnId") + .flatMap(hdfsNameNode) + .isEmpty + } + .map { nnId => + s"HDFS nameservice '$nameservice' declares NameNode '$nnId' but " + + s"dfs.namenode.rpc-address.$nameservice.$nnId is missing or not host:port" + } + } + } + } + val hasGlobalNameNode = catalogProperties + .get(HdfsNameNodeKey) + .exists(_.split(",").exists(_.trim.nonEmpty)) + def defaultFs = options + .get("fs.defaultFS") + .map(_.trim) + .filter(defaultFs => defaultFs.nonEmpty && hdfsNameNode(defaultFs).isEmpty) + .map { defaultFs => + s"the forwarded fs.defaultFS '$defaultFs' is not an HDFS host:port, which Comet's " + + "native HDFS client rejects for every path unless the catalog sets hdfs.name-node" + } + declarations.toSeq.headOption.orElse(if (hasGlobalNameNode) None else defaultFs) + } + /** * Transforms Hadoop S3A configuration keys to Iceberg FileIO property keys. * diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 9c708e2d0d1..e4e9ce900e1 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -89,7 +89,9 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // Loaded from the native storage factory: `builtin_storage_schemes` in // `native/core/src/execution/operators/iceberg_common.rs` is the single point of change and // explains why `oss` is read-only and `memory` write-only. Lazy so constructing the serde does - // not touch the native library. `gs` is additionally gated on the resolved FileIO below. + // not touch the native library. `gs` is additionally gated on the resolved FileIO below, and + // `hdfs` on its NameNode (`requireReachableHdfsNameNode`): the scheme alone does not make an + // HDFS location writable. private lazy val SupportedStorageSchemes: Set[String] = IcebergStorageSchemes.write // Supported schemes whose native backend is local and needs no host. Every other supported // scheme reads its bucket from the URL host (`requireSupportedStorageScheme`). @@ -427,6 +429,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { requirePositiveIntParquetSizes, requireNoParquetHadoopConfOverrides, requireSupportedStorageScheme, + requireReachableHdfsNameNode, requireSupportedHadoopS3Settings, requireSupportedS3FileIOProperties, requireGcsFileIOForGcsDataLocation, @@ -673,7 +676,9 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { if (!SupportedStorageSchemes.contains(scheme)) { Some(s"unsupported storage scheme: $scheme") } else if (!LocalStorageSchemes.contains(scheme) && !hasBucketAuthority(location)) { - Some(s"$scheme data location has no bucket in its authority: $location") + // For hdfs the authority is the NameNode or nameservice rather than a bucket. + val missing = if (scheme == "hdfs") "NameNode or nameservice" else "bucket" + Some(s"$scheme data location has no $missing in its authority: $location") } else { None } @@ -796,6 +801,38 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { unsupportedSettingsReason("S3 FileIO", unsupportedS3FileIOProperties(properties)) } + // The scheme rule above admits any `hdfs://` location with an authority, but native dials an + // authority with a port as is and resolves a portless one (a nameservice) only through its + // `hdfs.name-node.` declaration. A NameNode entry that is not host:port or a + // nameservice that did not resolve would pass planning and then fail on an executor, so decline + // them here. The judgement uses the properties `buildIcebergWriteProto` hands to native (see + // `hdfsCatalogProperties`), so the decision and the execution agree. + private val requireReachableHdfsNameNode: TriggerRule = ctx => + IcebergReflection.getDataLocation(ctx.table).filter(storageScheme(_) == "hdfs").flatMap { + location => + val dataUri = new java.net.URI(location) + val fileIOProperties = + IcebergReflection.getFileIOProperties(ctx.table).getOrElse(Map.empty[String, String]) + // Resolved as `buildIcebergWriteProto` does: the session configuration overlaid with the + // FileIO's (`effectiveHadoopConf`), so the decision and the executed declaration agree. + val resolutionConf = + CometIcebergNativeScan.hdfsResolutionConf(ctx.hadoopConf, Some(ctx.s3HadoopConf)) + CometIcebergNativeScan.hdfsNameNodeFallbackReason( + dataUri, + resolutionConf, + hdfsCatalogProperties(dataUri, resolutionConf, fileIOProperties)) + } + + // NameNode-related catalog properties for native: the declaration derived from the session + // Hadoop configuration first, so an explicit catalog `hdfs.name-node.` among the + // FileIO properties wins, as on the scan path. + private def hdfsCatalogProperties( + dataUri: java.net.URI, + hadoopConf: Configuration, + fileIOProperties: Map[String, String]): Map[String, String] = + CometIcebergNativeScan.hadoopToIcebergHdfsProperties(dataUri, hadoopConf) ++ + fileIOProperties + // HadoopFileIO takes its GCS configuration from `fs.gs.*`, which is not forwarded to the // native writer (only `fs.s3a.*` is bridged). Admit a gs:// data location only when the FileIO // Iceberg resolves for it is a GCSFileIO, whose `gcs.*` settings are forwarded. @@ -1162,8 +1199,15 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { val hadoopDerivedProperties = CometIcebergNativeScan.hadoopToIcebergS3Properties( NativeConfig.extractObjectStoreOptions(writeHadoopConf, dataUri), dataBucket) + // The HDFS properties are the ones `requireReachableHdfsNameNode` judged at planning time. + val hdfsResolutionConf = CometIcebergNativeScan.hdfsResolutionConf( + op.session.sessionState.newHadoopConf(), + Some(writeHadoopConf)) val catalogProperties = - hadoopDerivedProperties ++ fileIOProperties + CometIcebergNativeScan.ioTimeoutProperty() + hadoopDerivedProperties ++ hdfsCatalogProperties( + dataUri, + hdfsResolutionConf, + fileIOProperties) + CometIcebergNativeScan.ioTimeoutProperty() val common = IcebergWriteProtoTranslation.buildCommon( catalogProperties = catalogProperties, diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala new file mode 100644 index 00000000000..7a35509fcde --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/CometIcebergHdfsSuite.scala @@ -0,0 +1,597 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File +import java.net.URI +import java.nio.file.Files +import java.util.UUID + +import org.scalactic.source.Position +import org.scalatest.Tag + +import org.apache.commons.io.FileUtils +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} +import org.apache.hadoop.hdfs.{MiniDFSCluster, MiniDFSNNTopology} +import org.apache.hadoop.util.VersionInfo +import org.apache.spark.SparkConf +import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.comet.{CometIcebergNativeScanExec, CometIcebergWriteExec, IcebergCommitExec, IcebergWriteExec} +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper + +import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus +import org.apache.comet.iceberg.IcebergReflection + +/** + * End-to-end coverage of the native Iceberg scan and write against an `hdfs://` warehouse, backed + * by in-process `MiniDFSCluster`s. + * + * This is the only test that exercises iceberg-rust's `hdfs-native` backend for real. It matters + * because that backend is a second, independent HDFS client: the plain-Parquet native scan + * reaches HDFS through libhdfs/JNI (`fs.comet.libhdfs.schemes`), while an Iceberg table on HDFS + * is opened by a pure-Rust RPC client that shares nothing with it but the NameNode endpoints + * Comet hands it. A unit test over the scheme allowlists cannot tell whether that client actually + * connects, reads, or finishes a write. + * + * What is covered: + * - scans of tables on a single-NameNode cluster, whose locations carry a real `host:port` + * authority, so iceberg-rust needs no NameNode declaration; + * - native writes on the same cluster (unpartitioned and partitioned `INSERT`, a second + * `INSERT`, a copy-on-write `UPDATE`). Each asserts the `CometIcebergWriteExec` / + * `IcebergCommitExec` plan shape, that every committed file exists under the table's data + * location on HDFS with exactly the length its manifest entry records, that the file, row and + * byte counts `CometIcebergWriteExec` reports agree with what the NameNode holds, and that + * the native scan and plain Spark read the same rows back. The length check is the one that + * matters for `FileWrite::close`: the pinned iceberg-rust errors with "Wrote N bytes but + * storage reports M" when the two differ, and this is the first test to run it against a real + * NameNode; + * - a two-NameNode HA cluster addressed by nameservice (`hdfs:///...`), which is not a + * host, so the scan and the write only connect through the `hdfs.name-node.` declaration + * Comet derives from the session Hadoop configuration (the pinned iceberg-rust resolves a + * portless authority only through it). It is read before and after a failover. + * + * Which profiles run it. The cluster needs the `hadoop-client-minicluster` that `pom.xml` pins + * per Spark profile (`hadoop.version`: 3.4 -> 3.3.4, 3.5 -> 3.3.4, 4.0 -> 3.4.1, 4.1 -> 3.4.2, + * 4.2 -> 3.5.0) to match the `hadoop-client-api`/`runtime` Spark supplies. Measured by running + * the scan tests on every profile: + * - Spark 3.4, 3.5, 4.0 and 4.1: the cluster starts and every test runs, none canceled. A + * cluster that fails to start there is a broken fixture, so `beforeAll` fails the suite with + * the cause instead of cancelling the tests. + * - Spark 4.2: nothing runs (all tests are cancelled), for two independent reasons. There is no + * Iceberg runtime for 4.2 (`icebergAvailable` is false there, which alone cancelled the three + * original tests), and the cluster cannot start either: `MiniDFSCluster.Builder` throws + * `NoClassDefFoundError: org/junit/jupiter/api/Assertions`, because Hadoop 3.5.0's + * `GenericTestUtils` asserts with JUnit 5, which the shaded minicluster leaves out and the + * test classpath does not carry. With junit-jupiter-api 5.10.3 and its three small + * dependencies put on the JVM class path the cluster starts, so a test-scoped + * `junit-jupiter-api` dependency on the 4.2 profile is the fix once an Iceberg runtime + * exists. Until then this suite skips there and starts nothing. + */ +class CometIcebergHdfsSuite + extends CometTestBase + with AdaptiveSparkPlanHelper + with CometIcebergTestBase + with WithHdfsCluster { + + override protected def sparkConf: SparkConf = + super.sparkConf + // `UPDATE` on an Iceberg table is a row-level rewrite that these extensions plan. + .set( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") + + /** + * The only profile on which this suite does not run (see the class docstring): no Iceberg + * runtime, and no MiniDFSCluster. Anywhere else the Iceberg runtime and the cluster are + * required, and their absence fails the suite rather than cancelling it. + */ + private def profileSkipped: Boolean = isSpark42Plus + + override protected def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit + pos: Position): Unit = { + super.test(testName, testTags: _*) { + assume(!profileSkipped, "Spark 4.2 has no Iceberg runtime and no usable MiniDFSCluster") + testFun + } + } + + /** + * The nameservice of the HA cluster. Not the `ns1` of `simpleHATopology`, to be unmistakable. + */ + private val haNameservice = "cometha" + + private var haCluster: MiniDFSCluster = _ + private var haBaseDir: File = _ + + override def beforeAll(): Unit = { + super.beforeAll() + if (!profileSkipped) { + assert(icebergAvailable, "the Iceberg runtime is missing from the test classpath") + try startHdfsCluster() + catch { + case e @ (_: Exception | _: LinkageError) => + throw new IllegalStateException( + "MiniDFSCluster failed to start, so none of the HDFS tests can run. " + + s"Spark ${org.apache.spark.SPARK_VERSION}, hadoop-client-api " + + s"${VersionInfo.getVersion}, hadoop-client-minicluster loaded from " + + s"${classOf[MiniDFSCluster].getProtectionDomain.getCodeSource.getLocation}: $e", + e) + } + } + } + + override def afterAll(): Unit = { + try stopHaCluster() + finally { + try stopHdfsCluster() + finally super.afterAll() + } + } + + /** `hdfs://localhost:` -- the authority iceberg-rust dials as the NameNode. */ + private def hdfsUri: String = s"hdfs://localhost:$getDFSPort" + + private def assertSingleNativeScan(cometPlan: SparkPlan): CometIcebergNativeScanExec = { + val scans = collect(cometPlan) { case scan: CometIcebergNativeScanExec => scan } + assert( + scans.length == 1, + s"Expected exactly 1 CometIcebergNativeScanExec but found ${scans.length}. " + + s"Plan:\n$cometPlan") + scans.head + } + + /** + * Runs `f` with a Hadoop-catalog Iceberg warehouse rooted at `baseUri`, which is the + * single-NameNode cluster by default. The catalog name is unique per test so Spark's catalog + * cache cannot hand back a warehouse from an earlier test, and the warehouse directory is + * removed afterwards (`cleanup`). + */ + private def withHdfsIcebergCatalog(baseUri: String = hdfsUri, cleanup: Boolean = true)( + f: String => Unit): Unit = { + val catalog = s"hdfs_cat_${UUID.randomUUID().toString.replace("-", "")}" + val warehouse = s"$baseUri/warehouse/${UUID.randomUUID()}" + try { + withSQLConf( + s"spark.sql.catalog.$catalog" -> "org.apache.iceberg.spark.SparkCatalog", + s"spark.sql.catalog.$catalog.type" -> "hadoop", + s"spark.sql.catalog.$catalog.warehouse" -> warehouse, + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") { + f(catalog) + } + } finally { + if (cleanup) getFileSystem.delete(new Path(warehouse), true) + } + } + + /** + * Runs `sqlText` with Comet's native Iceberg writer enabled and returns the executed plan of + * every query that ran. Inline `VALUES` only reach the native writer if the local table scan is + * columnar too, hence the last setting. + */ + private def runNativeWrite(sqlText: String): Seq[SparkPlan] = { + // `withSQLConf` returns Unit before Spark 4.0, hence the var. + var plans: Seq[SparkPlan] = Seq.empty + withSQLConf( + CometConf.COMET_ICEBERG_WRITE_SPLIT_OPERATOR_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_WRITE_ENABLED.key -> "true", + CometConf.COMET_EXEC_LOCAL_TABLE_SCAN_ENABLED.key -> "true") { + plans = capturePlans(spark)(spark.sql(sqlText)) + } + plans + } + + private def nativeWrites(plans: Seq[SparkPlan]): Seq[CometIcebergWriteExec] = + plans.flatMap(p => collectWithSubqueries(p) { case w: CometIcebergWriteExec => w }) + + /** + * The write really went through Comet's native writer, in the split-operator shape: a native + * file writer, exactly one commit, and no row-based `IcebergWriteExec` left beside it. + */ + private def assertNativeWrite(plans: Seq[SparkPlan]): Unit = { + val jvmWrites = plans.flatMap(p => collectWithSubqueries(p) { case w: IcebergWriteExec => w }) + val commits = plans.flatMap(p => collectWithSubqueries(p) { case c: IcebergCommitExec => c }) + val dump = plans.mkString("\n--\n") + assert(nativeWrites(plans).nonEmpty, s"expected a CometIcebergWriteExec. Plans:\n$dump") + assert( + jvmWrites.isEmpty, + s"the JVM IcebergWriteExec ran beside the native one. Plans:\n$dump") + assert(commits.length == 1, s"expected exactly 1 IcebergCommitExec, got ${commits.length}") + } + + /** + * The native writer's own accounting of what it wrote agrees with the NameNode: the files, the + * rows, and the bytes. `written` are the files this write added, with their HDFS lengths. + */ + private def assertWriterReported( + plans: Seq[SparkPlan], + written: Seq[(Path, Long)], + rows: Long): Unit = { + def reported(metric: String): Long = nativeWrites(plans).map(_.metrics(metric).value).sum + assert(reported("numFiles") == written.size, s"numFiles vs $written") + assert(reported("numOutputRows") == rows, "numOutputRows") + assert( + reported("numOutputBytes") == written.map(_._2).sum, + s"the writer reports ${reported("numOutputBytes")} bytes, HDFS holds $written") + } + + private def tableLocation(catalog: String, table: String): String = { + val icebergTable = loadIcebergTable(spark, catalog, "db", table) + icebergTable.getClass.getMethod("location").invoke(icebergTable).toString + } + + private def snapshotCount(catalog: String, table: String): Long = + spark.sql(s"SELECT count(*) FROM $catalog.db.$table.snapshots").collect().head.getLong(0) + + /** Every data file the table's current snapshot references, with the size it records. */ + private def committedFiles(catalog: String, table: String): Seq[(Path, Long)] = + spark + .sql(s"SELECT file_path, file_size_in_bytes FROM $catalog.db.$table.files") + .collect() + .toSeq + .map(row => (new Path(row.getString(0)), row.getLong(1))) + + /** Every regular file under `root` on the cluster. */ + private def listFiles(fs: FileSystem, root: Path): Seq[FileStatus] = { + val it = fs.listFiles(root, true) + val files = Seq.newBuilder[FileStatus] + while (it.hasNext) files += it.next() + files.result() + } + + /** + * Checks the table's committed data files against what the NameNode and DataNode actually hold + * (read through `fs`), and returns them with their HDFS lengths. + * + * Every file the table references must exist under the table's `data/` directory with exactly + * the length its manifest entry records. That is the end-to-end check on iceberg-rust's + * `FileWrite::close`: the writer reports the byte count it wrote, and a `DataFile` that + * disagrees with storage would corrupt every later split-planning and rewrite. + * + * With `exact`, `data/` must hold nothing else either: no leftovers of an aborted attempt. A + * copy-on-write rewrite leaves the replaced files behind until they expire, so it opts out. + */ + private def assertCommittedFilesOnHdfs( + catalog: String, + table: String, + fs: FileSystem = getFileSystem, + exact: Boolean = true): Seq[(Path, Long)] = { + val dataDir = new Path(tableLocation(catalog, table), "data") + assert(dataDir.toUri.getScheme == "hdfs", s"expected an hdfs:// data location, got $dataDir") + + val committed = committedFiles(catalog, table) + assert(committed.nonEmpty, s"$catalog.db.$table references no data files") + val onHdfs = committed.map { case (path, recordedLength) => + assert( + path.toUri.getPath.startsWith(dataDir.toUri.getPath), + s"$path is not under the table's data directory $dataDir") + assert(recordedLength > 0, s"$path is recorded as empty") + val length = fs.getFileStatus(path).getLen + assert( + length == recordedLength, + s"$path: the manifest records $recordedLength bytes but HDFS holds $length") + (path, length) + } + + val onCluster = listFiles(fs, dataDir).map(_.getPath.toUri.getPath).toSet + val referenced = committed.map(_._1.toUri.getPath).toSet + assert(referenced.subsetOf(onCluster), s"missing from HDFS: ${referenced -- onCluster}") + if (exact) { + val extra = onCluster -- referenced + assert(extra.isEmpty, s"unreferenced files under $dataDir: $extra") + } + onHdfs + } + + /** The files in `after` that `before` did not have. */ + private def addedFiles( + before: Seq[(Path, Long)], + after: Seq[(Path, Long)]): Seq[(Path, Long)] = { + val known = before.map(_._1.toUri.getPath).toSet + after.filterNot { case (path, _) => known.contains(path.toUri.getPath) } + } + + test("native Iceberg scan reads a table stored on HDFS") { + withHdfsIcebergCatalog() { catalog => + spark.sql(s"CREATE TABLE $catalog.db.t (id INT, name STRING, value DOUBLE) USING iceberg") + spark.sql( + s"INSERT INTO $catalog.db.t VALUES (1, 'Alice', 10.5), (2, 'Bob', 20.3), " + + "(3, 'Charlie', 30.7)") + + // The data location must actually be on HDFS, or this suite would silently degrade into a + // duplicate of the local-filesystem coverage. + val dataLocation = IcebergReflection + .getDataLocation(loadIcebergTable(spark, catalog, "db", "t")) + .getOrElse(fail("could not resolve the Iceberg data location")) + assert( + dataLocation.startsWith("hdfs://"), + s"expected an hdfs:// data location, got $dataLocation") + + val (_, cometPlan) = checkSparkAnswer(s"SELECT * FROM $catalog.db.t ORDER BY id") + assertSingleNativeScan(cometPlan) + + spark.sql(s"DROP TABLE $catalog.db.t") + } + } + + test("native Iceberg scan on HDFS applies a pushed-down filter") { + withHdfsIcebergCatalog() { catalog => + spark.sql(s"CREATE TABLE $catalog.db.f (id INT, name STRING) USING iceberg") + spark.sql( + s"INSERT INTO $catalog.db.f VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')") + + val (_, cometPlan) = + checkSparkAnswer(s"SELECT id, name FROM $catalog.db.f WHERE id > 3 ORDER BY id") + assertSingleNativeScan(cometPlan) + + spark.sql(s"DROP TABLE $catalog.db.f") + } + } + + test("native Iceberg scan reads a partitioned table across multiple HDFS data files") { + withHdfsIcebergCatalog() { catalog => + spark.sql( + s"CREATE TABLE $catalog.db.p (id INT, part STRING) USING iceberg PARTITIONED BY (part)") + // Separate inserts so each partition lands in its own data file: a single-file read would + // not prove the operator cache serves more than one path from one NameNode. + spark.sql(s"INSERT INTO $catalog.db.p VALUES (1, 'x'), (2, 'x')") + spark.sql(s"INSERT INTO $catalog.db.p VALUES (3, 'y'), (4, 'y')") + + val (_, cometPlan) = checkSparkAnswer(s"SELECT * FROM $catalog.db.p ORDER BY id") + assertSingleNativeScan(cometPlan) + + spark.sql(s"DROP TABLE $catalog.db.p") + } + } + + test("native Iceberg INSERT writes its data files to HDFS and reads them back") { + withHdfsIcebergCatalog() { catalog => + spark.sql(s"CREATE TABLE $catalog.db.w (id INT, name STRING, value DOUBLE) USING iceberg") + + val plans = runNativeWrite( + s"INSERT INTO $catalog.db.w VALUES (1, 'Alice', 10.5), (2, 'Bob', 20.3), " + + "(3, 'Charlie', 30.7)") + assertNativeWrite(plans) + assert(snapshotCount(catalog, "w") == 1L) + assertWriterReported(plans, assertCommittedFilesOnHdfs(catalog, "w"), rows = 3) + + // Spark's own reader (Comet off) and the native scan agree on what was written. + val (_, cometPlan) = checkSparkAnswer(s"SELECT * FROM $catalog.db.w ORDER BY id") + assertSingleNativeScan(cometPlan) + val rows = spark.sql(s"SELECT id, name, value FROM $catalog.db.w ORDER BY id").collect() + assert( + rows.map(r => (r.getInt(0), r.getString(1), r.getDouble(2))).toSeq == + Seq((1, "Alice", 10.5), (2, "Bob", 20.3), (3, "Charlie", 30.7))) + } + } + + test("native Iceberg INSERT into a partitioned HDFS table, twice") { + withHdfsIcebergCatalog() { catalog => + spark.sql( + s"CREATE TABLE $catalog.db.pw (id INT, region STRING, amount DOUBLE) USING iceberg " + + "PARTITIONED BY (region)") + + val first = runNativeWrite( + s"INSERT INTO $catalog.db.pw VALUES (1, 'us-east', 10.5), (2, 'us-east', 20.3), " + + "(3, 'eu', 30.7)") + assertNativeWrite(first) + val afterFirst = assertCommittedFilesOnHdfs(catalog, "pw") + assertWriterReported(first, afterFirst, rows = 3) + + val second = + runNativeWrite(s"INSERT INTO $catalog.db.pw VALUES (4, 'eu', 1.5), (5, 'ap', 2.5)") + assertNativeWrite(second) + assert(snapshotCount(catalog, "pw") == 2L) + val afterSecond = assertCommittedFilesOnHdfs(catalog, "pw") + assertWriterReported(second, addedFiles(afterFirst, afterSecond), rows = 2) + + // The native writer created the partition directories on the cluster. + assert( + afterSecond.map(_._1.getParent.getName).toSet == + Set("region=us-east", "region=eu", "region=ap"), + s"unexpected partition directories: $afterSecond") + + val (_, cometPlan) = checkSparkAnswer(s"SELECT * FROM $catalog.db.pw ORDER BY id") + assertSingleNativeScan(cometPlan) + checkSparkAnswer(s"SELECT id FROM $catalog.db.pw WHERE region = 'eu' ORDER BY id") + } + } + + test("native Iceberg copy-on-write UPDATE rewrites its data files on HDFS") { + withHdfsIcebergCatalog() { catalog => + spark.sql( + s"CREATE TABLE $catalog.db.u (id INT, region STRING, amount DOUBLE) USING iceberg " + + "PARTITIONED BY (region) TBLPROPERTIES ('write.update.mode'='copy-on-write')") + val insert = runNativeWrite( + s"INSERT INTO $catalog.db.u VALUES (1, 'us-east', 10.0), (2, 'us-east', 20.0), " + + "(3, 'eu', 30.0)") + assertNativeWrite(insert) + val before = assertCommittedFilesOnHdfs(catalog, "u") + assertWriterReported(insert, before, rows = 3) + + // The scan feeding the rewrite and the writer behind it both talk to HDFS natively. + val update = + runNativeWrite(s"UPDATE $catalog.db.u SET amount = amount * 2 WHERE id = 2") + assertNativeWrite(update) + assert(snapshotCount(catalog, "u") == 2L) + val after = assertCommittedFilesOnHdfs(catalog, "u", exact = false) + + // Only the partition holding the updated row was rewritten: its two rows are in a new + // file, and the other partition's file is still the one the INSERT wrote. + val rewritten = addedFiles(before, after) + assert(rewritten.map(_._1.getParent.getName) == Seq("region=us-east"), s"$rewritten") + assertWriterReported(update, rewritten, rows = 2) + val untouched = after.map(_._1.toUri.getPath).toSet -- rewritten.map(_._1.toUri.getPath) + assert(untouched.size == 1 && untouched.head.contains("region=eu"), s"$untouched") + + val (_, cometPlan) = checkSparkAnswer(s"SELECT * FROM $catalog.db.u ORDER BY id") + assertSingleNativeScan(cometPlan) + val amounts = spark + .sql(s"SELECT id, amount FROM $catalog.db.u ORDER BY id") + .collect() + .map(r => (r.getInt(0), r.getDouble(1))) + .toSeq + assert(amounts == Seq((1, 10.0), (2, 40.0), (3, 30.0))) + } + } + + private def stopHaCluster(): Unit = { + try if (haCluster != null) haCluster.shutdown(true) + finally { + haCluster = null + if (haBaseDir != null) FileUtils.deleteQuietly(haBaseDir) + haBaseDir = null + } + } + + /** + * Starts a two-NameNode MiniDFSCluster with `nn1` active, and returns the client configuration + * a Spark job needs to address it by nameservice: exactly the keys a production `hdfs-site.xml` + * carries, so that Comet's derivation of `hdfs.name-node.` is what lets the native client + * dial. + */ + private def startHaCluster(): Seq[(String, String)] = { + val conf = new Configuration() + conf.set("dfs.namenode.metrics.logger.period.seconds", "0") + conf.set("dfs.datanode.metrics.logger.period.seconds", "0") + conf.setIfUnset("dfs.namenode.rpc-bind-host", "localhost") + // A second cluster in this JVM must not format the first one's `target/test/data/dfs`. + haBaseDir = Files.createTempDirectory("comet_ha_dfs_").toFile + conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, haBaseDir.getAbsolutePath) + + val topology = new MiniDFSNNTopology().addNameservice( + new MiniDFSNNTopology.NSConf(haNameservice) + .addNN(new MiniDFSNNTopology.NNConf("nn1")) + .addNN(new MiniDFSNNTopology.NNConf("nn2"))) + haCluster = new MiniDFSCluster.Builder(conf).nnTopology(topology).numDataNodes(1).build() + haCluster.transitionToActive(0) + + def rpcAddress(nn: String): String = { + val key = s"dfs.namenode.rpc-address.$haNameservice.$nn" + val address = haCluster.getConfiguration(0).get(key) + assert(address != null && address.nonEmpty, s"the HA cluster did not configure $key") + address + } + Seq( + "dfs.nameservices" -> haNameservice, + s"dfs.ha.namenodes.$haNameservice" -> "nn1,nn2", + s"dfs.namenode.rpc-address.$haNameservice.nn1" -> rpcAddress("nn1"), + s"dfs.namenode.rpc-address.$haNameservice.nn2" -> rpcAddress("nn2"), + s"dfs.client.failover.proxy.provider.$haNameservice" -> + "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider") + } + + test("native Iceberg scan and write resolve an HA nameservice and follow a failover") { + val haClientConf = startHaCluster() + val logicalUri = new URI(s"hdfs://$haNameservice") + val haConf = haClientConf.toMap + val expectedNameNodes = Seq("nn1", "nn2") + .map(nn => "hdfs://" + haConf(s"dfs.namenode.rpc-address.$haNameservice.$nn")) + .mkString(",") + + val hadoopConf = new Configuration() + haClientConf.foreach { case (k, v) => hadoopConf.set(k, v) } + try { + // The nameservice keys reach the session Hadoop configuration (planning, the Iceberg + // catalog, the commit) the way `spark.hadoop.*` settings would in a deployment. + withSQLConf(haClientConf: _*) { + withHdfsIcebergCatalog(baseUri = logicalUri.toString, cleanup = false) { catalog => + // A location with no host:port, which iceberg-rust cannot dial by itself. + spark.sql(s"CREATE TABLE $catalog.db.ha (id INT, name STRING) USING iceberg") + spark.sql(s"INSERT INTO $catalog.db.ha VALUES (1, 'a'), (2, 'b'), (3, 'c')") + val location = tableLocation(catalog, "ha") + assert( + location.startsWith(s"hdfs://$haNameservice/"), + s"expected a nameservice location, got $location") + val fs = FileSystem.get(logicalUri, hadoopConf) + // Written by the JVM writer, through the HA client. + val before = assertCommittedFilesOnHdfs(catalog, "ha", fs) + + // Succeeding is not enough: the declaration Comet derived is what the client was given. + // The per-nameservice key, since iceberg-rust reads the global `hdfs.name-node` only + // for authority-less paths. + def assertDeclared(properties: Map[String, String]): Unit = { + assert( + properties.get(s"hdfs.name-node.$haNameservice") == Some(expectedNameNodes), + s"$properties") + assert(!properties.contains("hdfs.name-node"), s"$properties") + } + + def scanAndCheck(): Unit = { + val query = s"SELECT * FROM $catalog.db.ha ORDER BY id" + // Checked on the planned scan before anything runs, so a lost declaration fails here + // with the properties in the message, not as a NameNode resolution error on an + // executor after the native client's retries. + assertDeclared( + assertSingleNativeScan( + spark + .sql(query) + .queryExecution + .executedPlan).nativeIcebergScanMetadata.catalogProperties) + val (_, cometPlan) = checkSparkAnswer(query) + assertDeclared( + assertSingleNativeScan(cometPlan).nativeIcebergScanMetadata.catalogProperties) + } + + scanAndCheck() + + // Fail over: nn1 goes standby and nn2 takes over. The first NameNode on the native + // client's list is now a standby, and the list must carry it to the new active. + haCluster.transitionToStandby(0) + haCluster.transitionToActive(1) + assert( + haCluster.getNameNode(0).isStandbyState && haCluster.getNameNode(1).isActiveState) + scanAndCheck() + + // The native writer finds the new active the same way, and what it writes is on the + // cluster with the length it reported. + val plans = runNativeWrite(s"INSERT INTO $catalog.db.ha VALUES (4, 'd')") + assertNativeWrite(plans) + val after = assertCommittedFilesOnHdfs(catalog, "ha", fs) + assertWriterReported(plans, addedFiles(before, after), rows = 1) + scanAndCheck() + assert( + spark.sql(s"SELECT count(*) FROM $catalog.db.ha").collect().head.getLong(0) == 4L) + + // A NameNode that is gone rather than standby: connections to nn1 are now refused, and + // both the cached read client and a fresh write client must still reach nn2. + haCluster.shutdownNameNode(0) + scanAndCheck() + val goneBefore = assertCommittedFilesOnHdfs(catalog, "ha", fs) + val gonePlans = runNativeWrite(s"INSERT INTO $catalog.db.ha VALUES (5, 'e')") + assertNativeWrite(gonePlans) + val goneAfter = assertCommittedFilesOnHdfs(catalog, "ha", fs) + assertWriterReported(gonePlans, addedFiles(goneBefore, goneAfter), rows = 1) + scanAndCheck() + assert( + spark.sql(s"SELECT count(*) FROM $catalog.db.ha").collect().head.getLong(0) == 5L) + } + } + } finally { + // Evict the cached client of the nameservice before its cluster goes away. + try FileSystem.get(logicalUri, hadoopConf).close() + catch { case _: Exception => () } + stopHaCluster() + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 7caffd0e7ee..f0c4e06a904 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -540,8 +540,82 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes dir, "bad_scheme", partitionSpec = "", - properties = Some("'write.data.path'='hdfs://nonexistent.invalid/iceberg/db/bad_scheme'")) - assertUnsupportedContainsAllowingWriteFailure("bad_scheme", "storage scheme", "hdfs") + // abfss: object_store reads it, but iceberg-rust's OpenDAL factory has no arm for it. + properties = + Some("'write.data.path'='abfss://c@nonexistent.invalid/iceberg/db/bad_scheme'")) + assertUnsupportedContainsAllowingWriteFailure("bad_scheme", "storage scheme", "abfss") + } + } + + test("Compatible when the data location scheme is hdfs") { + withDetectionCatalog { dir => + createTable( + dir, + "hdfs_scheme", + partitionSpec = "", + properties = + Some("'write.data.path'='hdfs://nonexistent.invalid:8020/iceberg/db/hdfs_scheme'")) + assertSupportLevelIs[Compatible]("hdfs_scheme", allowWriteFailure = true) + } + } + + test("fall-back: authority-less hdfs data location") { + // iceberg-rust would need `hdfs.name-node`, `hdfs.host`/`hdfs.port` or `hadoop.fs.defaultFS` + // for it, which Comet never relies on; the scheme rule declines it before the NameNode rule. + // Planned only, never executed: no filesystem on this classpath reaches it. + withDetectionCatalog { dir => + createTable( + dir, + "hdfs_no_authority", + partitionSpec = "", + properties = Some("'write.data.path'='hdfs:///iceberg/db/hdfs_no_authority'")) + assertUnsupportedContains( + planInsertWriteExec(s"$catalog.$ns.hdfs_no_authority"), + "hdfs_no_authority", + "hdfs data location has no NameNode or nameservice in its authority", + "hdfs:///") + } + } + + test("fall-back: hdfs data location in a nameservice that does not resolve to NameNodes") { + // `ns1` is a nameservice, not a host: without rpc-addresses to declare + // `hdfs.name-node.ns1` from, iceberg-rust could not resolve it and the task would fail on an + // executor. + withDetectionCatalog { dir => + withSQLConf("dfs.nameservices" -> "ns1", "dfs.ha.namenodes.ns1" -> "nn1,nn2") { + createTable( + dir, + "hdfs_unresolved_ns", + partitionSpec = "", + properties = Some("'write.data.path'='hdfs://ns1/iceberg/db/hdfs_unresolved_ns'")) + assertUnsupportedContains( + planInsertWriteExec(s"$catalog.$ns.hdfs_unresolved_ns"), + "hdfs_unresolved_ns", + "nameservice 'ns1' but Comet could not resolve it", + "hdfs.name-node.ns1") + } + } + } + + test("Compatible when the hdfs nameservice resolves to NameNodes") { + withDetectionCatalog { dir => + // Planned only, never executed: running the INSERT would make the JVM client dial `ns1` + // (a host name to it without a failover proxy provider, which some resolvers answer) or, + // with one, retry the unreachable NameNodes with backoff for minutes. + withSQLConf( + "dfs.nameservices" -> "ns1", + "dfs.ha.namenodes.ns1" -> "nn1,nn2", + "dfs.namenode.rpc-address.ns1.nn1" -> "nn1.nonexistent.invalid:8020", + "dfs.namenode.rpc-address.ns1.nn2" -> "nn2.nonexistent.invalid:8020") { + createTable( + dir, + "hdfs_resolved_ns", + partitionSpec = "", + properties = Some("'write.data.path'='hdfs://ns1/iceberg/db/hdfs_resolved_ns'")) + val support = CometIcebergNativeWrite.getSupportLevel( + planInsertWriteExec(s"$catalog.$ns.hdfs_resolved_ns")) + assert(support.isInstanceOf[Compatible], s"expected Compatible, got $support") + } } } @@ -561,6 +635,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes test("fall-back: hostless hdfs:/ data location is read as hdfs, not file") { // Hadoop normalises `hdfs:///p` to `hdfs:/p`; with no `://` the gate used to call it `file`. + // hdfs is a supported scheme on this branch, so the missing authority is what declines it. withDetectionCatalog { dir => createTable( dir, @@ -570,7 +645,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes assertUnsupportedContains( planInsertWriteExec(s"$catalog.$ns.hostless_hdfs"), "hostless_hdfs", - "unsupported storage scheme: hdfs") + "hdfs data location has no NameNode or nameservice in its authority") } } diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala index 21011fae65d..0ab359a96fe 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanSchemeFallbackSuite.scala @@ -25,12 +25,14 @@ import java.nio.file.Files import java.util.UUID import org.apache.commons.io.FileUtils +import org.apache.iceberg.{DataFiles, FileFormat, Table} import org.apache.spark.SparkConf import org.apache.spark.sql.{CometTestBase, SaveMode} -import org.apache.spark.sql.comet.{CometIcebergNativeScanExec, CometIcebergWriteExec, CometScanExec} +import org.apache.spark.sql.comet.{CometBatchScanExec, CometIcebergNativeScanExec, CometIcebergWriteExec, CometScanExec} import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.comet.{CometConf, CometIcebergTestBase, ExtendedExplainInfo, NativeBase} +import org.apache.comet.{CometConf, CometExplainInfo, CometIcebergTestBase, ExtendedExplainInfo, NativeBase} import org.apache.comet.hadoop.fs.{FakeHDFSFileSystem, FakeHdfsSchemeFileSystem, FakeWasbSchemeFileSystem} import org.apache.comet.iceberg.IcebergStorageSchemes @@ -188,7 +190,10 @@ class CometScanSchemeFallbackSuite extends CometTestBase with CometIcebergTestBa "s3://bucket/key.parquet", "s3a://bucket/key.parquet", "gs://bucket/key.parquet", - "oss://bucket/key.parquet").foreach { u => + "oss://bucket/key.parquet", + // hdfs-native backend, not the libhdfs/JNI client the plain-Parquet path uses. + "hdfs://nn:8020/warehouse/db/t/key.parquet", + "hdfs://nameservice1/warehouse/db/t/key.parquet").foreach { u => assert( CometScanRule.isIcebergReadableScheme(new URI(u), Set.empty), s"$u must be iceberg-readable; icebergReadableSchemes has regressed") @@ -299,6 +304,13 @@ class CometScanSchemeFallbackSuite extends CometTestBase with CometIcebergTestBa assert( !openable("blob:///bucket/k.parquet", Set.empty), "without opt-in, blob gets no bucket promotion, so a hostless blob location is unopenable") + // hdfs: the authority is the NameNode, and the gate runs before the catalog properties that + // could supply one, so a hostless location declines rather than guess. + assert(openable("hdfs://nn:8020/warehouse/db/t/k.parquet")) + assert(openable("hdfs://nameservice1/warehouse/db/t/k.parquet")) + assert( + !openable("hdfs:///warehouse/db/t/k.parquet"), + "authorityless hdfs:/// carries no NameNode and gets no promotion") } test("iceberg scan and write decline a wasb table instead of failing at execution") { @@ -389,4 +401,212 @@ class CometScanSchemeFallbackSuite extends CometTestBase with CometIcebergTestBa assert(sparkScans.isEmpty, s"expected no leftover Spark FileSourceScanExec:\n$transformed") } } + + // --- Iceberg scans over HDFS data files --------------------------------------------------- + // + // These drive the real planner gate (`CometScanRule` on a `BatchScanExec` over a real Iceberg + // table) rather than the static helpers above, because the NameNode decisions need the catalog + // properties the rule assembles from the session Hadoop configuration. The data files are only + // registered in the table's manifests, with the `hdfs://` paths the test names: planning reads + // manifests, never the data files, so nothing here needs a NameNode and nothing is executed. + + /** + * Plans `SELECT id` over an Iceberg table whose data files sit at `dataFiles`, applies + * `CometScanRule` with native Iceberg scans enabled and `hadoopConf` set on the session, and + * returns the scans the rule claimed plus the fallback reasons it left on the ones it declined. + */ + private def planIcebergScanOverHdfs( + dataFiles: Seq[String], + hadoopConf: Seq[(String, String)] = Seq.empty, + catalogProps: Seq[(String, String)] = Seq.empty): (Seq[CometBatchScanExec], Set[String]) = { + // A catalog of its own, so no test reads another's cached catalog or warehouse. + val catalog = s"hdfs_gate_${UUID.randomUUID().toString.take(8)}" + var claimed = Seq.empty[CometBatchScanExec] + var reasons = Set.empty[String] + // Catalog options set before the catalog's first use, so its FileIO is built with them. + val catalogConfs = catalogProps.map { case (k, v) => s"spark.sql.catalog.$catalog.$k" -> v } + withHadoopCatalog(catalog) { + withSQLConf(catalogConfs: _*) { + spark.sql(s"CREATE TABLE $catalog.db.t (id INT) USING iceberg") + val table = loadIcebergTable(spark, catalog, "db", "t").asInstanceOf[Table] + val append = table.newAppend() + dataFiles.foreach { path => + append.appendFile( + DataFiles + .builder(table.spec()) + .withPath(path) + .withFormat(FileFormat.PARQUET) + .withFileSizeInBytes(1024) + .withRecordCount(1) + .build()) + } + append.commit() + + // The Spark plan first, with Comet off, then the rule alone: no execution. + var sparkPlan: SparkPlan = null + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + sparkPlan = spark.sql(s"SELECT id FROM $catalog.db.t").queryExecution.executedPlan + } + val confs = Seq( + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") ++ hadoopConf + withSQLConf(confs: _*) { + val transformed = CometScanRule(spark).apply(stripAQEPlan(sparkPlan)) + claimed = transformed.collect { case s: CometBatchScanExec => s } + reasons = transformed + .collect { case s: BatchScanExec => + s.getTagValue(CometExplainInfo.FALLBACK_REASONS).getOrElse(Set.empty[String]) + } + .flatten + .toSet + } + } + } + (claimed, reasons) + } + + test("iceberg scan: a configured HDFS nameservice that does not resolve is declined") { + assume(icebergAvailable, "Iceberg not available in classpath") + // nn2 has no rpc-address, so the `hdfs.name-node.ns1` declaration is all-or-nothing empty and + // iceberg-rust could not resolve `ns1` on the executor. + val (claimed, reasons) = planIcebergScanOverHdfs( + Seq("hdfs://ns1/warehouse/db/t/a.parquet"), + Seq( + "dfs.nameservices" -> "ns1", + "dfs.ha.namenodes.ns1" -> "nn1,nn2", + "dfs.namenode.rpc-address.ns1.nn1" -> "nn1.example.com:8020")) + assert(claimed.isEmpty, "an unresolvable nameservice must not be claimed natively") + assert( + reasons.exists(_.contains("nameservice 'ns1' but Comet could not resolve it")), + s"expected the nameservice fall-back reason, got: $reasons") + } + + test("iceberg scan: a nameservice listed only in dfs.nameservices is declined") { + assume(icebergAvailable, "Iceberg not available in classpath") + val (claimed, reasons) = planIcebergScanOverHdfs( + Seq("hdfs://ns1/warehouse/db/t/a.parquet"), + Seq("dfs.nameservices" -> "ns1,ns2")) + assert(claimed.isEmpty, "a configured nameservice without NameNodes must not be claimed") + assert( + reasons.exists(_.contains("nameservice 'ns1' but Comet could not resolve it")), + s"expected the nameservice fall-back reason, got: $reasons") + } + + test("iceberg scan: a resolvable HDFS nameservice is claimed with its NameNode declaration") { + assume(icebergAvailable, "Iceberg not available in classpath") + // The control for the two tests above, and for the multi-authority one below: the same table + // shape is claimed once the nameservice resolves, so the declines are down to the gate under + // test. + val (claimed, reasons) = planIcebergScanOverHdfs( + Seq("hdfs://ns1/warehouse/db/t/a.parquet"), + Seq( + "dfs.nameservices" -> "ns1", + "dfs.ha.namenodes.ns1" -> "nn1,nn2", + "dfs.namenode.rpc-address.ns1.nn1" -> "nn1.example.com:9000", + "dfs.namenode.rpc-address.ns1.nn2" -> "hdfs://nn2.example.com:8020/")) + assert(claimed.size == 1, s"expected the scan to be claimed, fell back with: $reasons") + val properties = claimed.head.nativeIcebergScanMetadata.get.catalogProperties + // The per-nameservice key iceberg-rust resolves `hdfs://ns1` through; the global + // `hdfs.name-node` would only serve authority-less paths. + assert( + properties.get("hdfs.name-node.ns1") == + Some("hdfs://nn1.example.com:9000,hdfs://nn2.example.com:8020"), + s"unexpected catalog properties: $properties") + assert(!properties.contains("hdfs.name-node"), s"unexpected catalog properties: $properties") + } + + test("iceberg scan: a portless plain HDFS host is claimed on the default NameNode port") { + assume(icebergAvailable, "Iceberg not available in classpath") + // Not a configured nameservice, so it is a host the JVM dials on 8020; iceberg-rust has no + // default port, so Comet declares it. + val (claimed, reasons) = + planIcebergScanOverHdfs(Seq("hdfs://nn.example.com/warehouse/db/t/a.parquet")) + assert(claimed.size == 1, s"expected the scan to be claimed, fell back with: $reasons") + val properties = claimed.head.nativeIcebergScanMetadata.get.catalogProperties + assert( + properties.get("hdfs.name-node.nn.example.com") == Some("hdfs://nn.example.com:8020"), + s"unexpected catalog properties: $properties") + } + + test("iceberg scan: a plain host:port HDFS location is claimed with no NameNode property") { + assume(icebergAvailable, "Iceberg not available in classpath") + // Indistinguishable from a real NameNode, so the path authority is left to iceberg-rust. + val (claimed, reasons) = + planIcebergScanOverHdfs(Seq("hdfs://nn.example.com:8020/warehouse/db/t/a.parquet")) + assert(claimed.size == 1, s"expected the scan to be claimed, fell back with: $reasons") + assert( + !claimed.head.nativeIcebergScanMetadata.get.catalogProperties.keys + .exists(_.startsWith("hdfs.name-node"))) + } + + test("iceberg scan: data files on two HDFS authorities are declined") { + assume(icebergAvailable, "Iceberg not available in classpath") + // Comet declares the NameNodes of one HDFS authority per scan, so a second portless authority + // would reach native undeclared; the decline is conservative for host:port ones. It sits + // inline in the scan case of `CometScanRule.apply`, with no static helper of its own, so a real + // scan is the lowest layer that reaches it. + val (claimed, reasons) = planIcebergScanOverHdfs( + Seq( + "hdfs://nn1.example.com:8020/warehouse/db/t/a.parquet", + "hdfs://nn2.example.com:8020/warehouse/db/t/b.parquet")) + assert(claimed.isEmpty, "files on two NameNodes must not be claimed natively") + assert( + reasons.exists( + _.contains( + "across multiple HDFS authorities (nn1.example.com:8020, nn2.example.com:8020)")), + s"expected the multi-authority fall-back reason, got: $reasons") + } + + private val resolvableNs1 = Seq( + "dfs.nameservices" -> "ns1", + "dfs.ha.namenodes.ns1" -> "nn1,nn2", + "dfs.namenode.rpc-address.ns1.nn1" -> "nn1.example.com:8020", + "dfs.namenode.rpc-address.ns1.nn2" -> "nn2.example.com:8020") + + test("iceberg scan: a catalog hdfs.name-node. wins over the Hadoop-derived declaration") { + assume(icebergAvailable, "Iceberg not available in classpath") + val (claimed, reasons) = planIcebergScanOverHdfs( + Seq("hdfs://ns1/warehouse/db/t/a.parquet"), + resolvableNs1, + Seq("hdfs.name-node.ns1" -> "nn9.example.com:8020")) + assert(claimed.size == 1, s"expected the scan to be claimed, fell back with: $reasons") + val properties = claimed.head.nativeIcebergScanMetadata.get.catalogProperties + assert( + properties.get("hdfs.name-node.ns1") == Some("nn9.example.com:8020"), + s"the catalog value must reach native: $properties") + } + + test("iceberg scan: a catalog hdfs.name-node. entry without a port is declined") { + assume(icebergAvailable, "Iceberg not available in classpath") + // It would override the resolvable Hadoop-derived list and then fail on every executor. + val (claimed, reasons) = planIcebergScanOverHdfs( + Seq("hdfs://ns1/warehouse/db/t/a.parquet"), + resolvableNs1, + Seq("hdfs.name-node.ns1" -> "nn1.example.com")) + assert(claimed.isEmpty, "a portless catalog entry must not be claimed natively") + assert( + reasons.exists(_.contains("hdfs.name-node.ns1 entry 'nn1.example.com' is not host:port")), + s"expected the entry fall-back reason, got: $reasons") + } + + test("iceberg scan: an HA nameservice known only to the catalog's hadoop.* is declared") { + assume(icebergAvailable, "Iceberg not available in classpath") + // The JVM reader resolves it through the FileIO's configuration, which carries the catalog's + // `hadoop.*` overrides; deriving from the session configuration alone would mistake `nsb` for + // a plain host and declare `hdfs://nsb:8020`. + val (claimed, reasons) = planIcebergScanOverHdfs( + Seq("hdfs://nsb/warehouse/db/t/a.parquet"), + catalogProps = Seq( + "hadoop.dfs.nameservices" -> "nsb", + "hadoop.dfs.ha.namenodes.nsb" -> "nn1,nn2", + "hadoop.dfs.namenode.rpc-address.nsb.nn1" -> "a.example.com:8020", + "hadoop.dfs.namenode.rpc-address.nsb.nn2" -> "b.example.com:8020")) + assert(claimed.size == 1, s"expected the scan to be claimed, fell back with: $reasons") + val properties = claimed.head.nativeIcebergScanMetadata.get.catalogProperties + assert( + properties.get("hdfs.name-node.nsb") == + Some("hdfs://a.example.com:8020,hdfs://b.example.com:8020"), + s"unexpected catalog properties: $properties") + } } diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala index 6ff5daf27fe..776eaa45355 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala @@ -21,6 +21,8 @@ package org.apache.comet.serde.operator import java.lang.reflect.InvocationTargetException +import scala.jdk.CollectionConverters._ + import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers @@ -240,6 +242,496 @@ class CometIcebergNativeScanSuite extends AnyFunSuite with Matchers { out.values.toSet should not contain "https://some.example.com" } + // --- hadoopToIcebergHdfsProperties ------------------------------------------------------- + // + // These pin the declarations that make `hdfs:///...` reachable: the pinned + // iceberg-rust resolves a portless authority only through `hdfs.name-node.`; see + // that method's scaladoc. + + private def hadoopConfOf(conf: Map[String, String]): org.apache.hadoop.conf.Configuration = { + val hadoopConf = new org.apache.hadoop.conf.Configuration(false) + conf.foreach { case (k, v) => hadoopConf.set(k, v) } + hadoopConf + } + + private def hdfsProps(location: String, conf: Map[String, String]): Map[String, String] = + CometIcebergNativeScan.hadoopToIcebergHdfsProperties( + new java.net.URI(location), + hadoopConfOf(conf)) + + test("HA nameservice is declared with its NameNode list, in declaration order") { + val out = hdfsProps( + "hdfs://nameservice1/warehouse/db/t/metadata.json", + Map( + "dfs.ha.namenodes.nameservice1" -> "nn1,nn2", + "dfs.namenode.rpc-address.nameservice1.nn1" -> "host-a.example.com:8020", + "dfs.namenode.rpc-address.nameservice1.nn2" -> "host-b.example.com:8020")) + + // The per-nameservice key: iceberg-rust reads the global `hdfs.name-node` only for an + // authority-less path, so emitting that one would leave the nameservice undeclared. + out shouldBe Map( + "hdfs.name-node.nameservice1" -> + "hdfs://host-a.example.com:8020,hdfs://host-b.example.com:8020") + } + + test("a plain host:port authority needs no declaration") { + hdfsProps("hdfs://nn.example.com:8020/warehouse/db/t", Map.empty) shouldBe Map.empty + hdfsProps("hdfs://[::1]:8020/warehouse/db/t", Map.empty) shouldBe Map.empty + } + + test( + "a portless plain host is declared on the default NameNode RPC port, as the JVM dials it") { + // DFSUtilClient.getNNAddress applies 8020 to a portless `hdfs://host` URI; iceberg-rust has no + // default port and would treat the host as an undeclared nameservice. + hdfsProps("hdfs://nn.example.com/warehouse", Map.empty) shouldBe + Map("hdfs.name-node.nn.example.com" -> "hdfs://nn.example.com:8020") + // A portless IPv6 literal is not declared: iceberg-rust looks it up by the url crate's + // canonical spelling, which a raw Java authority need not match. The gate declines it. + hdfsProps("hdfs://[::1]/warehouse", Map.empty) shouldBe Map.empty + } + + test("a nameservice name with an underscore is read from the raw authority") { + // `URI.getHost` is null for it, which would silently derive nothing. + val conf = Map( + "dfs.ha.namenodes.name_service1" -> "nn1", + "dfs.namenode.rpc-address.name_service1.nn1" -> "h.example.com:8020") + hdfsProps("hdfs://name_service1/w", conf) shouldBe + Map("hdfs.name-node.name_service1" -> "hdfs://h.example.com:8020") + hdfsReason("hdfs://name_service1/w", conf, hdfsProps("hdfs://name_service1/w", conf)) shouldBe + None + } + + test("DNS-resolved NameNodes are not declared, since the native client would not expand them") { + val conf = Map( + "dfs.nameservices" -> "ns1", + "dfs.ha.namenodes.ns1" -> "nn", + "dfs.namenode.rpc-address.ns1.nn" -> "nn-dns.example.com:8020", + "dfs.client.failover.resolve-needed.ns1" -> "true") + hdfsProps("hdfs://ns1/warehouse", conf) shouldBe Map.empty + hdfsReason("hdfs://ns1/warehouse", conf, hdfsProps("hdfs://ns1/warehouse", conf)) + .getOrElse(fail("expected a reason")) should include( + "dfs.client.failover.resolve-needed.ns1") + } + + test("a partially resolved HA list yields nothing rather than a short failover list") { + // Dropping nn2 would silently turn a failover into an outage. + hdfsProps( + "hdfs://nameservice1/warehouse", + Map( + "dfs.ha.namenodes.nameservice1" -> "nn1,nn2", + "dfs.namenode.rpc-address.nameservice1.nn1" -> "host-a.example.com:8020")) shouldBe Map.empty + } + + test("a portless rpc-address leaves the nameservice unresolved, as the JVM rejects it") { + // Hadoop's HA client throws "Does not contain a valid host:port authority" for it, so Comet + // does not invent a port the JVM would not use. + for (address <- Seq("host-a.example.com", "hdfs://host-a.example.com", "[::1]")) { + withClue(s"rpc-address=$address: ") { + hdfsProps( + "hdfs://ns/warehouse", + Map( + "dfs.ha.namenodes.ns" -> "nn1,nn2", + "dfs.namenode.rpc-address.ns.nn1" -> address, + "dfs.namenode.rpc-address.ns.nn2" -> "host-b.example.com:9000")) shouldBe Map.empty + } + } + } + + test("rpc-addresses are normalized to hdfs://host:port") { + def resolved(address: String): Map[String, String] = + hdfsProps( + "hdfs://ns/warehouse", + Map("dfs.ha.namenodes.ns" -> "nn1", "dfs.namenode.rpc-address.ns.nn1" -> address)) + + // Not double-prefixed; a trailing slash and surrounding spaces dropped; IPv6 kept bracketed. + resolved("hdfs://host-a.example.com:8020") shouldBe + Map("hdfs.name-node.ns" -> "hdfs://host-a.example.com:8020") + resolved(" host-a.example.com:8020/ ") shouldBe + Map("hdfs.name-node.ns" -> "hdfs://host-a.example.com:8020") + resolved("[::1]:9000") shouldBe Map("hdfs.name-node.ns" -> "hdfs://[::1]:9000") + } + + test("a nameservice in dfs.nameservices without dfs.ha.namenodes yields nothing") { + // Not a plain host either, so no default port is applied to it. + hdfsProps("hdfs://ns2/warehouse", Map("dfs.nameservices" -> "ns1,ns2")) shouldBe Map.empty + } + + test("non-hdfs and authority-less locations are ignored") { + // Without the scheme check a bucket would look like a plain host and get `:8020`, and a + // resolvable "nameservice" would be declared for an S3 table. + hdfsProps("s3://bucket/key", Map.empty) shouldBe Map.empty + hdfsProps( + "s3://ns/key", + Map( + "dfs.ha.namenodes.ns" -> "nn1", + "dfs.namenode.rpc-address.ns.nn1" -> "h.example.com:8020")) shouldBe + Map.empty + hdfsProps("hdfs:///warehouse/db/t", Map.empty) shouldBe Map.empty + } + + // --- hdfsNameNodeFallbackReason ---------------------------------------------------------- + // + // The decision shared by the scan and the write planners: Some(reason) only when the pinned + // iceberg-rust could not resolve the NameNode with the properties native receives. + + private def hdfsReason( + location: String, + conf: Map[String, String] = Map.empty, + catalogProperties: Map[String, String] = Map.empty): Option[String] = + CometIcebergNativeScan.hdfsNameNodeFallbackReason( + new java.net.URI(location), + hadoopConfOf(conf), + catalogProperties) + + // What the planners pass: the Hadoop-derived declaration, then the catalog's properties. + private def plannedReason( + location: String, + conf: Map[String, String], + catalogProperties: Map[String, String] = Map.empty): Option[String] = + hdfsReason(location, conf, hdfsProps(location, conf) ++ catalogProperties) + + private val haConf = Map( + "dfs.nameservices" -> "ns1", + "dfs.ha.namenodes.ns1" -> "nn1,nn2", + "dfs.namenode.rpc-address.ns1.nn1" -> "host-a.example.com:8020", + "dfs.namenode.rpc-address.ns1.nn2" -> "host-b.example.com:8020") + + test("an authority-less hdfs location is declined, whatever else is configured") { + // iceberg-rust would open it from `hdfs.name-node`, `hdfs.host`/`hdfs.port` or `fs.defaultFS`, + // but Comet requires the NameNode in the location itself. + val reason = hdfsReason("hdfs:///warehouse/db/t") + reason.getOrElse(fail("expected a reason")) should startWith("authority-less hdfs location") + reason.get should include("hdfs:///warehouse/db/t") + hdfsReason( + "hdfs:///warehouse/db/t", + haConf ++ Map("fs.defaultFS" -> "hdfs://nn.example.com:8020"), + Map( + "hdfs.name-node" -> "nn.example.com:8020", + "hdfs.host" -> "nn.example.com", + "hdfs.port" -> "8020", + "hadoop.fs.defaultFS" -> "hdfs://nn.example.com:8020")).isDefined shouldBe true + } + + test("a resolved HA nameservice stays native") { + plannedReason("hdfs://ns1/warehouse", haConf) shouldBe None + } + + test("a portless plain host stays native through its derived declaration") { + plannedReason("hdfs://nn.example.com/warehouse", Map.empty) shouldBe None + // Without the declaration the same location is declined, so the derivation is what matters. + hdfsReason("hdfs://nn.example.com/warehouse").getOrElse(fail("expected a reason")) should + include("hdfs.name-node.nn.example.com") + } + + test("an authority with a port is dialed as is") { + hdfsReason("hdfs://nn.example.com:8020/warehouse") shouldBe None + hdfsReason("hdfs://[::1]:8020/warehouse") shouldBe None + // An authority that merely differs from the configured nameservice is just another host. + hdfsReason("hdfs://other.example.com:8020/warehouse", haConf) shouldBe None + } + + test("a configured nameservice that did not resolve is declined") { + val conf = haConf - "dfs.namenode.rpc-address.ns1.nn2" + val reason = plannedReason("hdfs://ns1/warehouse", conf) + reason.getOrElse(fail("expected a reason")) should include( + "HDFS location authority 'ns1' has no port") + reason.get should include("hdfs.name-node.ns1") + reason.get should include("could not resolve it") + reason.get should include("dfs.namenode.rpc-address.ns1.") + } + + test("a nameservice listed only in dfs.nameservices is declined") { + plannedReason("hdfs://ns2/warehouse", Map("dfs.nameservices" -> "ns1, ns2")) + .getOrElse(fail("expected a reason")) should include("nameservice 'ns2'") + // A nameservice that has only `dfs.ha.namenodes.` set is configured too. + plannedReason("hdfs://ns3/warehouse", Map("dfs.ha.namenodes.ns3" -> "nn1")) + .getOrElse(fail("expected a reason")) should include("nameservice 'ns3'") + } + + test("a catalog hdfs.name-node. declares an unresolved nameservice") { + val conf = haConf - "dfs.namenode.rpc-address.ns1.nn2" + plannedReason( + "hdfs://ns1/warehouse", + conf, + Map( + "hdfs.name-node.ns1" -> "host-a.example.com:8020,host-b.example.com:8020")) shouldBe None + } + + test("the global hdfs.name-node does not declare a nameservice") { + // iceberg-rust consults it only for authority-less paths, so it cannot rescue `hdfs://ns1`. + val conf = haConf - "dfs.namenode.rpc-address.ns1.nn2" + val reason = plannedReason( + "hdfs://ns1/warehouse", + conf, + Map("hdfs.name-node" -> "host-a.example.com:8020")) + reason.getOrElse(fail("expected a reason")) should include("has no port") + reason.get should include("hdfs.name-node.ns1") + // A blank global value is no value at all, not an invalid one. + hdfsReason( + "hdfs://nn.example.com:8020/w", + catalogProperties = Map("hdfs.name-node" -> " , ")) shouldBe + None + } + + test("a declaration for another nameservice does not declare this one") { + hdfsReason( + "hdfs://ns1/warehouse", + haConf - "dfs.namenode.rpc-address.ns1.nn2", + Map("hdfs.name-node.ns2" -> "host-a.example.com:8020")) + .getOrElse(fail("expected a reason")) should include("hdfs.name-node.ns1") + } + + test("Hadoop's own HA keys passed as hadoop.* declare a nameservice") { + val declared = Map( + "hadoop.dfs.ha.namenodes.ns9" -> "a,b", + "hadoop.dfs.namenode.rpc-address.ns9.a" -> "host-a.example.com:8020", + "hadoop.dfs.namenode.rpc-address.ns9.b" -> "hdfs://host-b.example.com:8020") + hdfsReason("hdfs://ns9/warehouse", catalogProperties = declared) shouldBe None + // iceberg-rust validates every declaration when it builds the storage, so an incomplete one + // fails all I/O: a NameNode whose rpc-address is missing or portless, or no NameNodes at all. + def invalid(props: Map[String, String], fragment: String): Unit = + hdfsReason("hdfs://ns9/warehouse", catalogProperties = props) + .getOrElse(fail(s"expected a reason for $props")) should include(fragment) + invalid( + declared - "hadoop.dfs.namenode.rpc-address.ns9.b", + "nameservice 'ns9' declares NameNode 'b' but dfs.namenode.rpc-address.ns9.b is missing") + invalid( + declared + ("hadoop.dfs.namenode.rpc-address.ns9.b" -> "host-b.example.com"), + "dfs.namenode.rpc-address.ns9.b is missing or not host:port") + invalid(Map("hadoop.dfs.ha.namenodes.ns9" -> " , "), "'ns9' is declared with no NameNodes") + // The hadoop.* keys win over the sugar key by key: the explicit id list stands, and the + // sugar's `nn0` rpc-address does not serve id `a`. + invalid( + Map( + "hadoop.dfs.ha.namenodes.ns9" -> "a", + "hdfs.name-node.ns9" -> "host-a.example.com:8020"), + "declares NameNode 'a'") + // ... but where the explicit ids are the sugar's own `nn`, its rpc-addresses fill them in. + hdfsReason( + "hdfs://ns9/warehouse", + catalogProperties = Map( + "hadoop.dfs.ha.namenodes.ns9" -> "nn0,nn1", + "hdfs.name-node.ns9" -> "host-a.example.com:8020,host-b.example.com:8020")) shouldBe None + } + + test("an incomplete declaration of another nameservice is declined for every location") { + // iceberg-rust validates every declared nameservice when it builds the storage. + hdfsReason( + "hdfs://nn.example.com:8020/w", + catalogProperties = Map("hadoop.dfs.ha.namenodes.other" -> "x")) + .getOrElse(fail("expected a reason")) should include( + "nameservice 'other' declares NameNode 'x'") + } + + test("a forwarded fs.defaultFS must be an HDFS host:port unless hdfs.name-node is set") { + // A logical default filesystem, as HA clusters write it, is rejected by iceberg-rust at + // storage build even when that nameservice is declared, failing every path. + val logical = Map( + "hadoop.fs.defaultFS" -> "hdfs://ns1", + "hdfs.name-node.ns1" -> "host-a.example.com:8020") + hdfsReason("hdfs://ns1/w", catalogProperties = logical) + .getOrElse(fail("expected a reason")) should include("fs.defaultFS 'hdfs://ns1'") + hdfsReason( + "hdfs://nn.example.com:8020/w", + catalogProperties = Map("hadoop.fs.defaultFS" -> "s3a://bucket")).isDefined shouldBe true + // The global hdfs.name-node takes precedence over it, so then it is never read. + hdfsReason( + "hdfs://ns1/w", + catalogProperties = logical + ("hdfs.name-node" -> "host-a.example.com:8020")) shouldBe None + // An HDFS host:port default filesystem, or a blank one, is fine. + for (defaultFs <- Seq("hdfs://nn.example.com:8020", "nn.example.com:8020/", " ")) { + withClue(s"fs.defaultFS=$defaultFs: ") { + hdfsReason( + "hdfs://nn.example.com:8020/w", + catalogProperties = Map("hadoop.fs.defaultFS" -> defaultFs)) shouldBe None + } + } + } + + test("a NameNode entry that is not host:port is declined, in any hdfs.name-node key") { + // iceberg-rust parses every one of them when it builds the storage, so one bad entry fails + // every read and write through it, even for a location that would not use it. + for { + key <- Seq("hdfs.name-node", "hdfs.name-node.ns1", "hdfs.name-node.other") + entry <- Seq( + "nn1.example.com", + "hdfs://nn1.example.com", + "hdfs://nn1.example.com/", + "[::1]", + "nn1.example.com:", + "nn1.example.com:http", + "nn1.example.com:0", + "nn1.example.com:99999", + "user@nn1.example.com:8020", + "nn1.example.com:8020/path") + } { + withClue(s"$key=$entry: ") { + val reason = hdfsReason( + "hdfs://nn9.example.com:8020/warehouse", + catalogProperties = Map(key -> entry)) + reason.getOrElse(fail("expected a reason")) should include(s"$key entry") + reason.get should include("is not host:port") + } + } + } + + test("one bad entry in a NameNode list is enough to decline it") { + val reason = hdfsReason( + "hdfs://ns1/warehouse", + catalogProperties = Map("hdfs.name-node.ns1" -> "nn1.example.com:8020, nn2.example.com")) + reason.getOrElse(fail("expected a reason")) should include( + "hdfs.name-node.ns1 entry 'nn2.example.com' is not host:port") + } + + test("a nameservice declaration must name a nameservice and list NameNodes") { + for (key <- Seq("hdfs.name-node.", "hdfs.name-node.n s")) { + withClue(s"$key: ") { + hdfsReason( + "hdfs://nn.example.com:8020/warehouse", + catalogProperties = Map(key -> "nn1.example.com:8020")) + .getOrElse(fail("expected a reason")) should include("does not name a nameservice") + } + } + hdfsReason( + "hdfs://nn.example.com:8020/warehouse", + catalogProperties = Map("hdfs.name-node.ns1" -> " , ")) + .getOrElse(fail("expected a reason")) should include("lists no NameNodes") + } + + test("well formed NameNode lists stay native") { + // As iceberg-rust reads them: entries trimmed, a trailing '/' dropped, `hdfs://` optional, + // a bracketed IPv6 literal allowed, and empty entries ignored. + for (value <- Seq( + "nn1.example.com:8020", + "hdfs://nn1.example.com:8020", + " hdfs://nn1.example.com:8020/ , nn2.example.com:9000 ", + "[::1]:8020,hdfs://[2001:db8::1]:8020", + "nn1.example.com:8020,")) { + withClue(s"hdfs.name-node.ns1=$value: ") { + hdfsReason("hdfs://ns1/warehouse", haConf, Map("hdfs.name-node.ns1" -> value)) shouldBe + None + } + } + } + + test("an authority iceberg-rust cannot parse, or might key differently, is declined") { + hdfsReason("hdfs://user@nn.example.com:8020/warehouse") + .getOrElse(fail("expected a reason")) should include("userinfo") + for (location <- Seq( + "hdfs://nn.example.com:0/warehouse", + "hdfs://nn.example.com:99999/warehouse", + // Portless IPv6 and non-ASCII names: iceberg-rust's url crate rewrites them, so a + // declaration keyed by the raw spelling could miss. + "hdfs://[::1]/warehouse", + "hdfs://ns%C3%A9/warehouse")) { + withClue(s"$location: ") { + hdfsReason(location).getOrElse(fail("expected a reason")) should include( + "neither host:port") + } + } + } + + test("non-hdfs locations never get an HDFS reason") { + val badProps = Map("hdfs.name-node" -> "no-port", "hdfs.name-node.ns1" -> "no-port") + for (location <- Seq( + "s3://ns1/key", + "s3a://ns1/key", + "gs://ns1/key", + "file:///tmp/warehouse", + "/tmp/warehouse", + "memory://ns1/key")) { + withClue(s"$location: ") { + hdfsReason(location, haConf - "dfs.namenode.rpc-address.ns1.nn2", badProps) shouldBe None + } + } + } + + test("the hdfs scheme is matched case-insensitively") { + hdfsReason("HDFS:///warehouse").isDefined shouldBe true + hdfsReason("HDFS://ns1/warehouse", Map("dfs.nameservices" -> "ns1")).isDefined shouldBe true + } + + test("other HDFS settings iceberg-rust rejects for every path are declined") { + // iceberg-rust parses these when it builds the storage, before resolving any path. + for (props <- Seq( + Map("hdfs.port" -> "80x"), + Map("hdfs.port" -> "70000"), + Map("hdfs.host" -> "nn.example.com:8020"), + Map("hdfs.host" -> "hdfs://nn.example.com"), + Map("hdfs.host" -> "nn.example.com", "hdfs.port" -> "0"), + Map("hadoop." -> "x"))) { + withClue(s"$props: ") { + hdfsReason("hdfs://nn.example.com:8020/w", catalogProperties = props).isDefined shouldBe + true + } + } + // Valid ones, and blank ones, are left alone. + for (props <- Seq( + Map("hdfs.host" -> "nn.example.com", "hdfs.port" -> "9000"), + Map("hdfs.host" -> "::1"), + Map("hdfs.port" -> " "), + Map("hdfs.user" -> "hdfs"))) { + withClue(s"$props: ") { + hdfsReason("hdfs://nn.example.com:8020/w", catalogProperties = props) shouldBe None + } + } + } + + test("declaring a nameservice named like the native client's synthetic one is declined") { + // opendal builds every client against a synthetic HA nameservice called `nameservice`, and a + // forwarded declaration of a real one by that name replaces its NameNodes. + for (props <- Seq( + Map("hdfs.name-node.nameservice" -> "nn1.example.com:8020"), + Map( + "hadoop.dfs.ha.namenodes.nameservice" -> "a", + "hadoop.dfs.namenode.rpc-address.nameservice.a" -> "nn1.example.com:8020"))) { + withClue(s"$props: ") { + hdfsReason( + "hdfs://ns1/w", + catalogProperties = props + ("hdfs.name-node.ns1" -> + "nn2.example.com:8020")).getOrElse(fail("expected a reason")) should include( + "synthetic nameservice") + // The location's own nameservice by that name is consistent with itself. + hdfsReason("hdfs://nameservice/w", catalogProperties = props) shouldBe None + } + } + } + + test("NameNodes resolve from the session configuration overlaid with the FileIO's") { + val session = hadoopConfOf( + Map( + "dfs.nameservices" -> "ns1", + "dfs.ha.namenodes.ns1" -> "nn1", + "dfs.namenode.rpc-address.ns1.nn1" -> "session.example.com:8020", + "dfs.ha.namenodes.ns2" -> "nn1", + "dfs.namenode.rpc-address.ns2.nn1" -> "only-session.example.com:8020", + "fs.defaultFS" -> "hdfs://session.example.com:8020")) + val fileIO = hadoopConfOf( + Map( + "dfs.namenode.rpc-address.ns1.nn1" -> "catalog.example.com:8020", + "dfs.ha.namenodes.nsb" -> "nn1", + "dfs.namenode.rpc-address.nsb.nn1" -> "only-catalog.example.com:8020")) + val conf = CometIcebergNativeScan.hdfsResolutionConf(session, Some(fileIO)) + + // The FileIO's catalog overrides win; what only one side knows is kept from it. + hdfsProps("hdfs://ns1/w", confMap(conf)) shouldBe + Map("hdfs.name-node.ns1" -> "hdfs://catalog.example.com:8020") + hdfsProps("hdfs://ns2/w", confMap(conf)) shouldBe + Map("hdfs.name-node.ns2" -> "hdfs://only-session.example.com:8020") + hdfsProps("hdfs://nsb/w", confMap(conf)) shouldBe + Map("hdfs.name-node.nsb" -> "hdfs://only-catalog.example.com:8020") + // Only `dfs.*` settings take part. + conf.get("fs.defaultFS") shouldBe null + // No FileIO configuration: the session's alone. + hdfsProps( + "hdfs://ns2/w", + confMap(CometIcebergNativeScan.hdfsResolutionConf(session, None))) shouldBe + Map("hdfs.name-node.ns2" -> "hdfs://only-session.example.com:8020") + } + + private def confMap(conf: org.apache.hadoop.conf.Configuration): Map[String, String] = + conf.getPropsWithPrefix("").asScala.toMap + private def keyMetadataMethod(clazz: Class[_]) = clazz.getMethod("keyMetadata") private def serializeDeleteFile(file: AnyRef) =